diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 87a65214ab..2ce47ea714 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -28,7 +28,7 @@ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go # UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /usr/local/cargo/bin/uv /usr/local/bin/uv ENV TZ=Etc/UTC diff --git a/.github/workflows/backend-test-windows.yml b/.github/workflows/backend-test-windows.yml index 54a62ec495..3468449753 100644 --- a/.github/workflows/backend-test-windows.yml +++ b/.github/workflows/backend-test-windows.yml @@ -63,7 +63,7 @@ jobs: - uses: denoland/setup-deno@v2 with: - # Pin to the Deno version shipped in the runtime image (Dockerfile) so CI + # Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 @@ -81,7 +81,7 @@ jobs: - uses: astral-sh/setup-uv@v7 with: - version: "0.11.24" + version: "0.13.0" - uses: shivammathur/setup-php@v2 with: diff --git a/.github/workflows/backend-test.yml b/.github/workflows/backend-test.yml index a2087025b7..cdfda7e0d4 100644 --- a/.github/workflows/backend-test.yml +++ b/.github/workflows/backend-test.yml @@ -50,7 +50,7 @@ jobs: dotnet-version: "9.0.x" - uses: denoland/setup-deno@v2 with: - # Pin to the Deno version shipped in the runtime image (Dockerfile) so CI + # Pin to the Deno version shipped in the full runtime image (docker/DockerfileFull) so CI # tests what production runs, instead of floating on the latest v2.x. deno-version: 2.2.1 - uses: actions/setup-go@v7 @@ -64,7 +64,7 @@ jobs: node-version: "24" - uses: astral-sh/setup-uv@v7 with: - version: "0.11.24" + version: "0.13.0" - uses: shivammathur/setup-php@v2 with: php-version: "8.3" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index d420ff1f00..475d7860d8 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -27,7 +27,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl @@ -111,7 +111,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl @@ -125,7 +125,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -136,7 +136,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -147,7 +147,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -188,7 +188,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl @@ -202,7 +202,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -213,7 +213,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -224,7 +224,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -235,7 +235,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -246,7 +246,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -257,7 +257,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always @@ -268,7 +268,7 @@ jobs: LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} MODE: worker WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always steps: @@ -309,7 +309,7 @@ jobs: DATABASE_URL: postgres://postgres:changeme@postgres:5432/windmill LICENSE_KEY: ${{ secrets.WM_LICENSE_KEY_CI }} WORKER_GROUP: main - WORKER_TAGS: deno,bun,go,python3,bash,dependency,flow,nativets + WORKER_TAGS: bun,go,python3,bash,dependency,flow,nativets options: >- --pull always --health-interval 10s --health-timeout 5s --health-retries 5 --health-cmd "curl diff --git a/.release-please-manifest.json b/.release-please-manifest.json index cc72934156..fdce702331 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.828.0" + ".": "1.830.0" } diff --git a/AGENTS.md b/AGENTS.md index e4915cdcb6..fd4b60a64b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,8 +37,9 @@ Open-source platform for internal tools, workflows, API integrations, background - **Operator write rights**: `docs/operator-write-rights.md` — which `operator_settings` flags are enforced rather than cosmetic, and why a right that is granted-unless-withdrawn needs `Option` fields and a jsonb merge rather than a serde default -- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace setting that lets - operators compose flows, what the composition check must cover, and why it costs a full seat +- **Operator builder rights**: `docs/operator-builder-rights.md` — the workspace settings that let + operators compose flows and full-code apps, what the composition check must cover, which + raw-app endpoint stays closed on purpose, and why either right costs a full seat - **Auth surface**: `docs/auth-surface.md` — credential precedence, session/cache invalidation scope, which token labels email their owner at expiry, how OAuth login matches `login_type`, and that every superadmin route refuses `$WM_TOKEN`. Read before designing anything that creates diff --git a/CHANGELOG.md b/CHANGELOG.md index c958457c7a..80939b1b47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,55 @@ # Changelog +## [1.830.0](https://github.com/windmill-labs/windmill/compare/v1.829.0...v1.830.0) (2026-10-09) + + +### Features + +* add a jobs:run:dependencies token scope for lock generation ([#11643](https://github.com/windmill-labs/windmill/issues/11643)) ([1dc4ed5](https://github.com/windmill-labs/windmill/commit/1dc4ed5e7fb86c9658f1f877e7b92cd9dab5e283)) +* let operators compose full-code apps when the workspace grants it ([#11229](https://github.com/windmill-labs/windmill/issues/11229)) ([02e4697](https://github.com/windmill-labs/windmill/commit/02e4697157112581e9e2a9303180077126db714b)) +* operators get AI sessions ([#11443](https://github.com/windmill-labs/windmill/issues/11443)) ([338c4e9](https://github.com/windmill-labs/windmill/commit/338c4e921dc3185d886c8b040f253285adf20efb)) +* replace the hub_sync script with an opt-in daily server sync ([#11624](https://github.com/windmill-labs/windmill/issues/11624)) ([2b55a8a](https://github.com/windmill-labs/windmill/commit/2b55a8afbc8a86c6cebe97b1cab02c313b986f22)) +* retry transient AI provider failures before output reaches readers ([#11623](https://github.com/windmill-labs/windmill/issues/11623)) ([d3ddef0](https://github.com/windmill-labs/windmill/commit/d3ddef0b7ec4705aa578c4eca24b96e9152485bd)) +* stop offering deno for new scripts, ship it only on full images ([#11648](https://github.com/windmill-labs/windmill/issues/11648)) ([cda7883](https://github.com/windmill-labs/windmill/commit/cda78830fde1b96bd3de2674e23c245b50e3c051)) + + +### Bug Fixes + +* **cli:** count superadmins as able to preserve ownership on push ([#11653](https://github.com/windmill-labs/windmill/issues/11653)) ([70db567](https://github.com/windmill-labs/windmill/commit/70db567a78b1e1bc9e2dcc9ff2139951d778117e)) +* harden app secret validation ([#11655](https://github.com/windmill-labs/windmill/issues/11655)) ([8556a7b](https://github.com/windmill-labs/windmill/commit/8556a7bebe093e0bd61c33d67415fd553f0070ad)) +* open the chat + menu without waiting on the skills and mcp fetches ([#11645](https://github.com/windmill-labs/windmill/issues/11645)) ([1fea1aa](https://github.com/windmill-labs/windmill/commit/1fea1aa6609d644e02bac8d16186f9a2dfcb63cd)) +* say Running on a chat run card only once its job exists ([#11627](https://github.com/windmill-labs/windmill/issues/11627)) ([d046187](https://github.com/windmill-labs/windmill/commit/d046187e3edd01535240c6080b483afe154ae189)) + +## [1.829.0](https://github.com/windmill-labs/windmill/compare/v1.828.0...v1.829.0) (2026-10-09) + + +### Features + +* add an api base url instance setting for displayed urls ([#11628](https://github.com/windmill-labs/windmill/issues/11628)) ([6acf185](https://github.com/windmill-labs/windmill/commit/6acf1858540e83b65b5218c50c98af692092d22e)) +* add Pi template to the AI sandbox picker ([#11619](https://github.com/windmill-labs/windmill/issues/11619)) ([a85d0b7](https://github.com/windmill-labs/windmill/commit/a85d0b76cc494aaa4a484a273ec534bdc2958102)) +* agent nits: multi-select, memory-off notice, agent step label ([#11622](https://github.com/windmill-labs/windmill/issues/11622)) ([c52ba2f](https://github.com/windmill-labs/windmill/commit/c52ba2fd30e19986522420ff02fae224a075b4f6)) +* export worker cpu, memory and occupancy metrics ([#11629](https://github.com/windmill-labs/windmill/issues/11629)) ([5d90f96](https://github.com/windmill-labs/windmill/commit/5d90f96863b64e9bc88ed35429f8068a6a788d1e)) +* let a job see that its worker is draining ([#11612](https://github.com/windmill-labs/windmill/issues/11612)) ([588d846](https://github.com/windmill-labs/windmill/commit/588d8463dfa1c8936b8753b5f0fd794ecb115203)) +* let workspace admins delete other users' drafts ([#11644](https://github.com/windmill-labs/windmill/issues/11644)) ([68376fc](https://github.com/windmill-labs/windmill/commit/68376fc011ef6406a820c69217058316514d3f8d)) +* per-script resource rollup and heaviest scripts table ([#11631](https://github.com/windmill-labs/windmill/issues/11631)) ([442357d](https://github.com/windmill-labs/windmill/commit/442357dbe8db0c52995572acd7385dedf4efdc42)) +* pipeline editor: assets-only view, Fix actions and draft triggers ([#11440](https://github.com/windmill-labs/windmill/issues/11440)) ([3dae553](https://github.com/windmill-labs/windmill/commit/3dae553576ba23b2c1f56e4deac01305c4645c01)) + + +### Bug Fixes + +* advertise the actual signing algorithm in OIDC discovery ([#11637](https://github.com/windmill-labs/windmill/issues/11637)) ([98fc65a](https://github.com/windmill-labs/windmill/commit/98fc65a16bbe2bc9515cdd5443304053c4322274)) +* ask a workspace for its session backups before pushing to it ([#11641](https://github.com/windmill-labs/windmill/issues/11641)) ([2ea846f](https://github.com/windmill-labs/windmill/commit/2ea846f563f5a77cbd3eae2eddee9e38a3700d00)) +* keep instance settings users sub-tabs from being squashed ([#11633](https://github.com/windmill-labs/windmill/issues/11633)) ([fb22e5c](https://github.com/windmill-labs/windmill/commit/fb22e5ce368eba9146e0d51b8fdabb52ca9f68f0)) +* keep raw app deploy panel settings shared with the editor ([#11616](https://github.com/windmill-labs/windmill/issues/11616)) ([0a95ea4](https://github.com/windmill-labs/windmill/commit/0a95ea47facaede2545cc4741d4a70c1c847e11a)) +* load schedules once per page load, poll critical alerts every 30s ([#11639](https://github.com/windmill-labs/windmill/issues/11639)) ([1b4e686](https://github.com/windmill-labs/windmill/commit/1b4e686b2f5de3138df0c3b6ed627ea0fe0a2f7d)) +* name the open workspace settings tab in the page header breadcrumb ([#11617](https://github.com/windmill-labs/windmill/issues/11617)) ([e233908](https://github.com/windmill-labs/windmill/commit/e233908c1e99a246dc7c94fa86ea82c5cbe9d6b3)) +* **python:** add missing import-to-package mappings for common modules ([#11638](https://github.com/windmill-labs/windmill/issues/11638)) ([ab8606f](https://github.com/windmill-labs/windmill/commit/ab8606f37921913ef1410da735abb5742c025225)) +* send hub script completions to the workspace handlers ([#11613](https://github.com/windmill-labs/windmill/issues/11613)) ([d685332](https://github.com/windmill-labs/windmill/commit/d685332771dcdfc718d7a2077fe1cdb3b323909b)) +* set WM_END_USER_EMAIL on direct script and flow runs ([#11636](https://github.com/windmill-labs/windmill/issues/11636)) ([2bdeceb](https://github.com/windmill-labs/windmill/commit/2bdeceba7fb6815cf3150ecb856afb8eb3872e44)) +* store a WAC parent's first start and wall duration ([#11625](https://github.com/windmill-labs/windmill/issues/11625)) ([8f28a58](https://github.com/windmill-labs/windmill/commit/8f28a5860595dbe558e6777a06bb97b6ec6a3e67)) +* sync back volume files rewritten to the same size ([#11620](https://github.com/windmill-labs/windmill/issues/11620)) ([2758920](https://github.com/windmill-labs/windmill/commit/2758920f1182ddd12e6473a644a4a966a435522e)) +* uniform decrypt error and clear stored credentials on logout ([#11632](https://github.com/windmill-labs/windmill/issues/11632)) ([987c321](https://github.com/windmill-labs/windmill/commit/987c321ebcd305df79fa945e941abc2b2e1d3e95)) + ## [1.828.0](https://github.com/windmill-labs/windmill/compare/v1.827.0...v1.828.0) (2026-10-08) diff --git a/Dockerfile b/Dockerfile index b1b3f939e7..c6a4ccc55f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -83,7 +83,6 @@ COPY /python-client/docs/ /frontend/static/pydocs/ RUN npm run generate-backend-client ENV NODE_OPTIONS "--max-old-space-size=8192" -ARG VITE_BASE_URL "" # Must be declared for the build-arg to reach the bundle. See frontend/svelte.config.js. ARG WM_BUILD_VERSION="" # Read more about macro in docker/dev.nu @@ -242,7 +241,7 @@ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtimes to temp build location (will copy with world-writable perms later) # --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run @@ -287,8 +286,6 @@ COPY --from=builder /frontend/build /static_frontend COPY --from=builder /windmill/target/release/windmill ${APP}/windmill COPY --from=windmill_duckdb_ffi_internal_builder /windmill-duckdb-ffi-internal/target/release/libwindmill_duckdb_ffi_internal.so ${APP}/libwindmill_duckdb_ffi_internal.so -COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno - COPY --from=oven/bun:1.4.0 /usr/local/bin/bun /usr/bin/bun # Install windmill CLI diff --git a/README.md b/README.md index 87fc66f399..e9ed6381f1 100644 --- a/README.md +++ b/README.md @@ -150,7 +150,7 @@ You can run scripts locally by passing the right environment variables for the ` - **Frontend**: Svelte 5 - **Sandboxing**: [nsjail](https://github.com/google/nsjail) and PID namespace isolation - **Runtimes**: - - TypeScript/JavaScript: Bun (default) and Deno + - TypeScript/JavaScript: Bun (default); Deno only on the `windmill-full` images - Python: python3 with uv for dependency management - Go, Bash, PowerShell, PHP, Rust, C#, Java, Ansible @@ -318,7 +318,7 @@ running options. This will also avoid compile time issue with sqlx's `query!` macro. 2. (optional, linux only) Install [nsjail](https://github.com/google/nsjail) and have it accessible in your PATH -3. Install bun, deno and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun`,`/usr/bin/deno`, and +3. Install bun and python3 (+ any languages you want to use), have the bins at `/usr/bin/bun` and `/usr/local/bin/python3` or set the corresponding environment variables. 4. (optional) Install the [lld linker](https://lld.llvm.org/) 5. Go to `frontend/`: diff --git a/backend/.sqlx/query-782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde.json b/backend/.sqlx/query-009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6.json similarity index 66% rename from backend/.sqlx/query-782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde.json rename to backend/.sqlx/query-009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6.json index f3449712c9..c687589db1 100644 --- a/backend/.sqlx/query-782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde.json +++ b/backend/.sqlx/query-009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)\n VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error(\"fail\"); }', 'deno', 'script', 'test-user', '{}', '', '', '', true)\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted)\n VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error(\"fail\"); }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n', true)\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "782c259ccdf71c5e2fa8dcb27515e7c5ff83009774dc15095ed0dbcb0ecdbdde" + "hash": "009fd85c1aa55ea36454eafffbf62505461eb1cc8b9af85a2e2ab6e7dbe950c6" } diff --git a/backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json b/backend/.sqlx/query-1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e.json similarity index 62% rename from backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json rename to backend/.sqlx/query-1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e.json index 6e5f8645bf..a0847ae2ca 100644 --- a/backend/.sqlx/query-dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730.json +++ b/backend/.sqlx/query-1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'bun', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "dbc05cbf6720cc85e01118bcdcbb478724ac0ea8f3e043da2de9d932216cc730" + "hash": "1043947475822c27f73b5357e6ec0d527d999f2865ff5de75186b49004a3565e" } diff --git a/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json b/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json new file mode 100644 index 0000000000..1946f55366 --- /dev/null +++ b/backend/.sqlx/query-1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null,\n extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras\n ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at))\n END\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "int8", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1333436e3d0466eccdaf99fb70aadad56b50619036fab3f2c1bdbbf280a80217" +} diff --git a/backend/.sqlx/query-1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8.json b/backend/.sqlx/query-1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8.json new file mode 100644 index 0000000000..6ae002d4f7 --- /dev/null +++ b/backend/.sqlx/query-1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM runnable_job_stats s WHERE s.hour >= $1 AND NOT EXISTS (SELECT 1 FROM workspace w WHERE w.id = s.workspace_id)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "1ae4aab40783b4cd588e5a4da57eabca765789cd706ad682e8094c8960bc46a8" +} diff --git a/backend/.sqlx/query-1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80.json b/backend/.sqlx/query-1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80.json new file mode 100644 index 0000000000..988ff44bb4 --- /dev/null +++ b/backend/.sqlx/query-1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE((operator_settings->>'builder_flows')::boolean, false) AS \"flows!\",\n COALESCE((operator_settings->>'builder_apps')::boolean, false) AS \"apps!\",\n COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\",\n COALESCE((operator_settings->>'manage_triggers')::boolean, true) AS \"triggers!\"\n FROM workspace_settings WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "flows!", + "type_info": "Bool" + }, + { + "ordinal": 1, + "name": "apps!", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "schedules!", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "triggers!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null, + null, + null, + null + ] + }, + "hash": "1fd3d5e8918143c6e4828b092c5d7d376555217b037d7694efa5ceffc4980f80" +} diff --git a/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json b/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json deleted file mode 100644 index 04b23832d9..0000000000 --- a/backend/.sqlx/query-212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at)\n VALUES ('admins', $1, $2, $3, $4, $6, now())\n ON CONFLICT (workspace_id, name) DO UPDATE\n SET schema = EXCLUDED.schema, description = EXCLUDED.description,\n -- A fileset is a set of files, so it cannot also be one file.\n -- Create and update reject the pair; this writer bypasses both, so\n -- it declines the extension rather than persisting the forbidden\n -- combination onto a same-named local fileset.\n format_extension = CASE\n WHEN resource_type.is_fileset THEN NULL\n WHEN $5 THEN EXCLUDED.format_extension\n ELSE resource_type.format_extension END,\n display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END,\n edited_at = now()", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Varchar", - "Jsonb", - "Text", - "Varchar", - "Bool", - "Varchar", - "Bool" - ] - }, - "nullable": [] - }, - "hash": "212bf5b32de102a9c537907aaff04befc3a6596c258086805dd42f513ddb3ead" -} diff --git a/backend/.sqlx/query-24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875.json b/backend/.sqlx/query-24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875.json new file mode 100644 index 0000000000..eaa7d60dea --- /dev/null +++ b/backend/.sqlx/query-24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO runnable_job_stats\n (hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms,\n max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms)\n SELECT s.hour, s.workspace_id, s.runnable_path, s.worker_group, s.job_count,\n s.total_duration_ms, s.max_memory_peak, s.sum_memory_peak,\n s.memory_sample_count, s.total_cpu_ms\n FROM UNNEST($1::bigint[], $2::text[], $3::text[], $4::text[], $5::int[], $6::bigint[],\n $7::int[], $8::bigint[], $9::int[], $10::bigint[])\n AS s(hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms,\n max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms)\n JOIN workspace w ON w.id = s.workspace_id\n ORDER BY s.hour, s.workspace_id, s.runnable_path, s.worker_group\n ON CONFLICT (hour, workspace_id, runnable_path, worker_group) DO UPDATE SET\n job_count = runnable_job_stats.job_count + EXCLUDED.job_count,\n total_duration_ms = runnable_job_stats.total_duration_ms + EXCLUDED.total_duration_ms,\n max_memory_peak = GREATEST(runnable_job_stats.max_memory_peak, EXCLUDED.max_memory_peak),\n sum_memory_peak = runnable_job_stats.sum_memory_peak + EXCLUDED.sum_memory_peak,\n memory_sample_count = runnable_job_stats.memory_sample_count + EXCLUDED.memory_sample_count,\n total_cpu_ms = runnable_job_stats.total_cpu_ms + EXCLUDED.total_cpu_ms\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8Array", + "TextArray", + "TextArray", + "TextArray", + "Int4Array", + "Int8Array", + "Int4Array", + "Int8Array", + "Int4Array", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "24cfd719a45af5beb4eb264939290c01e1b684525bae4cb329515989cd341875" +} diff --git a/backend/.sqlx/query-7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877.json b/backend/.sqlx/query-344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d.json similarity index 69% rename from backend/.sqlx/query-7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877.json rename to backend/.sqlx/query-344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d.json index 90b559a9b6..37f090c2b4 100644 --- a/backend/.sqlx/query-7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877.json +++ b/backend/.sqlx/query-344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "7f98c013682aca6d269ef6ef754a7247b6e2dd4958d0968a3f2eb49cc320e877" + "hash": "344c08104d62d9bcd3f9117abb574e73a6fb364e4ab3e3916958c2e493673b1d" } diff --git a/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json b/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json deleted file mode 100644 index 2f794e6be0..0000000000 --- a/backend/.sqlx/query-386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "exists", - "type_info": "Bool" - } - ], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Text", - "Text", - "Bool", - "Text", - "Bool" - ] - }, - "nullable": [ - null - ] - }, - "hash": "386e14cf7572027f2c4ef313cd7cc5dd6c7b0da4f76131313d3e6d97f36c5c34" -} diff --git a/backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json b/backend/.sqlx/query-42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779.json similarity index 58% rename from backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json rename to backend/.sqlx/query-42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779.json index 532fc430ab..402a406820 100644 --- a/backend/.sqlx/query-2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95.json +++ b/backend/.sqlx/query-42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'bun', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "2d43c1889b492df6f77380e739e41e20fe6bea0553cc39e30b7aa8ae59963f95" + "hash": "42250637493a623850ca76fe5a705fe4acef039f38a5e76da8d09f2f7e755779" } diff --git a/backend/.sqlx/query-fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4.json b/backend/.sqlx/query-69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927.json similarity index 59% rename from backend/.sqlx/query-fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4.json rename to backend/.sqlx/query-69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927.json index c747e99795..751b727d0f 100644 --- a/backend/.sqlx/query-fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4.json +++ b/backend/.sqlx/query-69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'deno', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'bun', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "fe17aff543f28cf4d6d09ea7c4c7d18ff6a95a8402468131f5a5f33df91c59e4" + "hash": "69e1c04c246c42a3a1d7202b6fb349d0ac779431993e225945ddfb91489cb927" } diff --git a/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json b/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json new file mode 100644 index 0000000000..ba1cc13fac --- /dev/null +++ b/backend/.sqlx/query-71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e.json @@ -0,0 +1,45 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at?", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "last_run_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Bool", + "Text", + "Bool", + "Varchar", + "Text", + "Bool", + "Int4", + "Int8", + "TextArray", + "Uuid", + "Text" + ] + }, + "nullable": [ + true, + false, + null + ] + }, + "hash": "71c9e0c0f1d9eba0f24c8e410ca2f2a64dc559e3228d3b763f68b1ad3491ae6e" +} diff --git a/backend/.sqlx/query-51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e.json b/backend/.sqlx/query-8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0.json similarity index 61% rename from backend/.sqlx/query-51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e.json rename to backend/.sqlx/query-8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0.json index 6eceb86132..3420b8c337 100644 --- a/backend/.sqlx/query-51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e.json +++ b/backend/.sqlx/query-8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return \"ok\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -10,5 +10,5 @@ }, "nullable": [] }, - "hash": "51f18c817d4014e847486f5a8dbf205ffa0e53d5bd3fdb97777e988389b1d69e" + "hash": "8511e096394edb1433f6572d50b66d8558866af53d829d83fa634f12a90733c0" } diff --git a/backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json b/backend/.sqlx/query-8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3.json similarity index 54% rename from backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json rename to backend/.sqlx/query-8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3.json index e103a91c1e..53a9daaf9a 100644 --- a/backend/.sqlx/query-7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec.json +++ b/backend/.sqlx/query-8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2", + "query": "WITH RECURSIVE chain(id, parent_job) AS (\n SELECT id, parent_job FROM v2_job WHERE id = $1 AND workspace_id = $2\n UNION ALL\n SELECT j.id, j.parent_job FROM v2_job j\n JOIN chain c ON j.id = c.parent_job AND j.workspace_id = $2\n )\n SELECT\n CASE WHEN a.agent THEN regexp_replace(j.runnable_path, '\\.chat$', '')\n ELSE j.runnable_path END AS runnable_path,\n CASE\n WHEN a.agent THEN 'agents'\n WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts'\n WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows'\n WHEN j.kind IN ('dependencies', 'flowdependencies') THEN 'dependencies'\n WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN\n CASE WHEN COALESCE(\n (SELECT m->'value'->>'type'\n FROM jsonb_array_elements(j.raw_flow->'modules') m\n WHERE m->>'id' IN ('a', 'main')\n LIMIT 1),\n 'script'\n ) = 'flow' THEN 'flows' ELSE 'scripts' END\n END AS scope_kind,\n CASE WHEN j.trigger_kind = 'app' THEN j.trigger END AS launched_by_app\n FROM v2_job j JOIN chain c ON c.id = j.id,\n LATERAL (SELECT j.kind = 'flowpreview'\n AND j.raw_flow->'modules'->1 IS NULL\n AND j.raw_flow->'modules'->0->>'id' = '__wm_agent_root' AS agent) a\n WHERE j.workspace_id = $2", "describe": { "columns": [ { @@ -31,5 +31,5 @@ null ] }, - "hash": "7904ac35748b1de2c7ff764f22787436d0fe74f99f298e5c399233f0d247f2ec" + "hash": "8aef4f9d27ec6df07cd671bcf90bf378e9fac4986cd29f69c1f2b8a83ba515c3" } diff --git a/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json b/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json deleted file mode 100644 index f105dd7825..0000000000 --- a/backend/.sqlx/query-9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Bool", - "Text", - "Bool", - "Varchar", - "Text", - "Bool", - "Int4", - "Int8", - "TextArray" - ] - }, - "nullable": [ - false - ] - }, - "hash": "9212902d4eb2834ab3b429968a7a85c53c49e4ef0f42c146324d5c71fbd1e21d" -} diff --git a/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json b/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json deleted file mode 100644 index 0b166b32a4..0000000000 --- a/backend/.sqlx/query-975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5.json +++ /dev/null @@ -1,33 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, d.started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000),\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING duration_ms\n ), parent_ping AS (\n UPDATE v2_job_runtime r SET ping = now()\n FROM v2_job_queue q\n WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12\n AND EXISTS (SELECT 1 FROM completed)\n )\n SELECT duration_ms AS \"duration_ms!\" FROM completed", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "duration_ms!", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid", - "Bool", - "Text", - "Bool", - "Varchar", - "Text", - "Bool", - "Int4", - "Int8", - "TextArray", - "Uuid", - "Text" - ] - }, - "nullable": [ - false - ] - }, - "hash": "975158ae72ec12f3f933fba1d9623bf1c925feebe746299cd1f0eda3f8b949d5" -} diff --git a/backend/.sqlx/query-97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e.json b/backend/.sqlx/query-97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e.json new file mode 100644 index 0000000000..abf8b56d79 --- /dev/null +++ b/backend/.sqlx/query-97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM runnable_job_stats WHERE hour < $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "97d8a9835344b8c72d59afc99d76b681c5120ef29eb09b8626a3fbc43a32cc0e" +} diff --git a/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json b/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json new file mode 100644 index 0000000000..59ed0e1767 --- /dev/null +++ b/backend/.sqlx/query-9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null,\n extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras\n ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at))\n END\n WHERE id = $1 AND workspace_id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Text", + "Int4", + "Float8" + ] + }, + "nullable": [] + }, + "hash": "9e7a5e1ebba42217092971dd340cd9182a4812b14735e502958eaafeb4c6b33d" +} diff --git a/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json b/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json deleted file mode 100644 index 2358af807f..0000000000 --- a/backend/.sqlx/query-a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "email", - "type_info": "Varchar" - }, - { - "ordinal": 1, - "name": "executions", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [ - false, - null - ] - }, - "hash": "a2f6ca89faaa8d8739f67cd1aae571de793b377dd6058065f51b12c974e5b665" -} diff --git a/backend/.sqlx/query-d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c.json b/backend/.sqlx/query-b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a.json similarity index 61% rename from backend/.sqlx/query-d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c.json rename to backend/.sqlx/query-b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a.json index ec3ae54f17..a5a8c00494 100644 --- a/backend/.sqlx/query-d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c.json +++ b/backend/.sqlx/query-b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'deno', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'bun', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -11,5 +11,5 @@ }, "nullable": [] }, - "hash": "d2a9a9af4d00c1c05276c14e6f32044437800a4b0c60aa364ab2f362a4fd9a2c" + "hash": "b0c10d17ed684af4dc79db7a199d4438ea003b5762178f25ff12ef23c1a8f46a" } diff --git a/backend/.sqlx/query-b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37.json b/backend/.sqlx/query-b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37.json new file mode 100644 index 0000000000..d65f75636d --- /dev/null +++ b/backend/.sqlx/query-b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b159cc3ab6fddcfb561c90d20bee2ff605a1c3721a4cbcc080f98fbb87277c37" +} diff --git a/backend/.sqlx/query-c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477.json b/backend/.sqlx/query-c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477.json new file mode 100644 index 0000000000..d84578d37c --- /dev/null +++ b/backend/.sqlx/query-c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477.json @@ -0,0 +1,74 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT workspace_id, runnable_path, worker_group,\n SUM(job_count)::bigint AS \"job_count!\",\n SUM(total_duration_ms)::bigint AS \"total_duration_ms!\",\n MAX(max_memory_peak) AS \"max_memory_peak!\",\n SUM(sum_memory_peak)::bigint AS \"sum_memory_peak!\",\n SUM(memory_sample_count)::bigint AS \"memory_sample_count!\",\n SUM(total_cpu_ms)::bigint AS \"total_cpu_ms!\"\n FROM runnable_job_stats\n WHERE hour >= $1\n AND ($2::text IS NULL OR worker_group = $2)\n AND ($3::text IS NULL OR workspace_id = $3)\n GROUP BY workspace_id, runnable_path, worker_group\n ORDER BY CASE $4::text\n WHEN 'max_memory' THEN MAX(max_memory_peak)::numeric\n WHEN 'cpu' THEN SUM(total_cpu_ms)\n ELSE SUM(total_duration_ms)\n END DESC, workspace_id, runnable_path, worker_group\n LIMIT $5\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "worker_group", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "job_count!", + "type_info": "Int8" + }, + { + "ordinal": 4, + "name": "total_duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "max_memory_peak!", + "type_info": "Int4" + }, + { + "ordinal": 6, + "name": "sum_memory_peak!", + "type_info": "Int8" + }, + { + "ordinal": 7, + "name": "memory_sample_count!", + "type_info": "Int8" + }, + { + "ordinal": 8, + "name": "total_cpu_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Text", + "Text", + "Text", + "Int8" + ] + }, + "nullable": [ + false, + false, + false, + null, + null, + null, + null, + null, + null + ] + }, + "hash": "c5971326bdbfaa7dff74689d1365db4f26c41fa56b326dbd02e604c793a13477" +} diff --git a/backend/.sqlx/query-cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20.json b/backend/.sqlx/query-cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74.json similarity index 69% rename from backend/.sqlx/query-cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20.json rename to backend/.sqlx/query-cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74.json index a68e5f1253..91cacf4123 100644 --- a/backend/.sqlx/query-cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20.json +++ b/backend/.sqlx/query-cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'deno', 'script', 'test-user', '{}', '', '', '')\n ", + "query": "\n INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock)\n VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return \"handled\"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\\n//bun.lock\\n')\n ", "describe": { "columns": [], "parameters": { @@ -8,5 +8,5 @@ }, "nullable": [] }, - "hash": "cdc594b30855e9384481f561b2c16200d4e6dacb1eb274513c8f1795235dfe20" + "hash": "cc719da4da98b6fe9612a4cc2f42dd19834a3e0d0be593ea6bc1239fe5453a74" } diff --git a/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json b/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json new file mode 100644 index 0000000000..ba7a89ef73 --- /dev/null +++ b/backend/.sqlx/query-ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8.json @@ -0,0 +1,43 @@ +{ + "db_name": "PostgreSQL", + "query": "WITH deleted AS (\n DELETE FROM v2_job_queue WHERE id = $1\n RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason,\n (extras->>'wac_first_started_at')::timestamptz AS first_started_at,\n COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms\n ), completed AS (\n INSERT INTO v2_job_completed AS cj\n ( workspace_id\n , id\n , started_at\n , duration_ms\n , extras\n , result\n , result_columns\n , canceled_by\n , canceled_reason\n , flow_status\n , workflow_as_code_status\n , memory_peak\n , status\n , worker\n )\n SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at),\n CASE WHEN d.first_started_at IS NULL THEN d.run_ms\n ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END,\n CASE WHEN d.first_started_at IS NOT NULL\n THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END,\n $3::text::jsonb, $10,\n CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END,\n CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END,\n s.flow_status, s.workflow_as_code_status, $8,\n CASE WHEN $4::BOOL OR d.canceled_by IS NOT NULL THEN 'canceled'::job_status\n WHEN $7::BOOL THEN 'skipped'::job_status\n WHEN $2::BOOL THEN 'success'::job_status\n ELSE 'failure'::job_status END,\n d.worker\n FROM deleted d LEFT JOIN v2_job_status s ON s.id = d.id\n ON CONFLICT (id) DO UPDATE SET status = EXCLUDED.status, result = $3::text::jsonb,\n canceled_by = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_by ELSE cj.canceled_by END,\n canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL\n THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END\n RETURNING cj.started_at, cj.duration_ms,\n COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms\n )\n SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\",\n last_run_ms AS \"last_run_ms!\" FROM completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "started_at?", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "duration_ms!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "last_run_ms!", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Uuid", + "Bool", + "Text", + "Bool", + "Varchar", + "Text", + "Bool", + "Int4", + "Int8", + "TextArray" + ] + }, + "nullable": [ + true, + false, + null + ] + }, + "hash": "ce39fd9650500b16f7247c8173a40c7e5d2d8a51aea2d36f2bc8c6f86d53a9b8" +} diff --git a/backend/.sqlx/query-d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8.json b/backend/.sqlx/query-d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8.json new file mode 100644 index 0000000000..55b335fba0 --- /dev/null +++ b/backend/.sqlx/query-d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM runnable_job_stats WHERE workspace_id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "d2fbaaaea6c542342862d8b535673d19dc4ca721ab656a39330bad5ea48999d8" +} diff --git a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json b/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json deleted file mode 100644 index ae893b6791..0000000000 --- a/backend/.sqlx/query-e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1)\n UPDATE v2_job_queue q SET running = false, started_at = null\n FROM prev WHERE q.id = $1\n RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "int8", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Uuid" - ] - }, - "nullable": [ - null - ] - }, - "hash": "e12f852fa196d16862151ab490b05e6f5c25e23b7c41ce1c0e5a83bf7b118bfd" -} diff --git a/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json b/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json new file mode 100644 index 0000000000..e22953610d --- /dev/null +++ b/backend/.sqlx/query-e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT usr.email, usage.executions\n FROM usr, LATERAL (\n SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions\n FROM v2_job_completed c JOIN v2_job j USING (id)\n WHERE j.workspace_id = $1\n AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow')\n AND j.permissioned_as_email = usr.email\n AND now() - '1 week'::interval < j.created_at\n ) usage\n WHERE workspace_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "executions", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + null + ] + }, + "hash": "e8779f1cddc475554dbbd2e3122db8a21a2eaacf971bf046cef1ee14cf04e323" +} diff --git a/backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json b/backend/.sqlx/query-efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b.json similarity index 58% rename from backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json rename to backend/.sqlx/query-efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b.json index a2a87b680d..81505d64a3 100644 --- a/backend/.sqlx/query-cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b.json +++ b/backend/.sqlx/query-efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT COALESCE((ws.operator_settings->>'builder_flows')::boolean, false)) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", + "query": "WITH active_users as (SELECT distinct username as email FROM audit_partitioned WHERE timestamp > NOW() - INTERVAL '1 month' AND (operation = 'users.login' OR operation = 'oauth.login' OR operation = 'users.token.refresh') AND username NOT IN (SELECT email FROM usr WHERE is_service_account)),\n active_authors as (SELECT distinct t.email FROM usr t LEFT JOIN workspace_settings ws ON ws.workspace_id = t.workspace_id WHERE NOT (t.operator AND NOT (COALESCE((ws.operator_settings->>'builder_flows')::boolean, false) OR COALESCE((ws.operator_settings->>'builder_apps')::boolean, false))) AND t.email IN (SELECT email FROM active_users)),\n active_authors_agg as (SELECT array_agg(email) as authors FROM active_authors),\n active_ops_agg as (SELECT array_agg(email) as operators from active_users WHERE email NOT IN (SELECT email FROM active_authors))\n SELECT active_authors_agg.authors, active_ops_agg.operators, array_length(active_authors_agg.authors, 1) as author_count, array_length(active_ops_agg.operators, 1) as operator_count FROM active_authors_agg, active_ops_agg", "describe": { "columns": [ { @@ -34,5 +34,5 @@ null ] }, - "hash": "cbc521b505c3953dc624ae6cefe6999e6d0c732da83ebc74ef7678cacd22ad6b" + "hash": "efc7aab19d729ea419c1fcaef47122096921dce2dac3979a8b788bf76f32646b" } diff --git a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json b/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json deleted file mode 100644 index 5f86fb97d8..0000000000 --- a/backend/.sqlx/query-f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE v2_job_queue\n SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null\n WHERE id = $1 AND workspace_id = $2", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Uuid", - "Text", - "Int4", - "Float8" - ] - }, - "nullable": [] - }, - "hash": "f0498c9bddc0d4d8948167f3ed849bf8b3523d8dde1ef6b8f1950423830cc6ed" -} diff --git a/backend/.sqlx/query-f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18.json b/backend/.sqlx/query-f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18.json new file mode 100644 index 0000000000..0f57f5d951 --- /dev/null +++ b/backend/.sqlx/query-f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18.json @@ -0,0 +1,59 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM draft\n WHERE workspace_id = $1 AND path = $2 AND typ = $3\n AND ($4::text IS NULL OR email = $4)\n RETURNING 1 as \"one!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "one!", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "draft_kind", + "kind": { + "Enum": [ + "script", + "flow", + "app", + "raw_app", + "resource", + "variable", + "trigger_schedule", + "trigger_webhook", + "trigger_default_email", + "trigger_email", + "trigger_http", + "trigger_websocket", + "trigger_postgres", + "trigger_kafka", + "trigger_nats", + "trigger_mqtt", + "trigger_sqs", + "trigger_gcp", + "trigger_azure", + "trigger_poll", + "trigger_cli", + "trigger_nextcloud", + "trigger_google", + "trigger_github", + "data_pipeline", + "trigger_amqp" + ] + } + } + }, + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "f338741f0866dc717ccfcf6b7d46b4e55ca07e472610b29fad24b64f45f98e18" +} diff --git a/backend/AGENTS.md b/backend/AGENTS.md index 750e0594e1..fbacb857ee 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -214,7 +214,7 @@ Whole classes of defect compile and unit-test clean: - **Stack overflow from a large buffer in an async block.** An array declared across an `.await` is baked into the future's state; once that future is boxed a few layers deep by the job poller, two 16 KB arrays abort the worker *process* (`thread - 'tokio-runtime-worker' has overflowed its stack`). Heap-allocate read buffers + 'tokio-rt-worker' has overflowed its stack`). Heap-allocate read buffers (`vec![0u8; N]`, not `[0u8; N]`). - Deadlocks from draining only one of a child's pipes, missed cancellation or timeout propagation, and anything depending on the real engine's output format. diff --git a/backend/Cargo.lock b/backend/Cargo.lock index b5f3d9a306..aa3d601492 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 4 -[[package]] -name = "addr2line" -version = "0.25.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" -dependencies = [ - "gimli", -] - [[package]] name = "adler2" version = "2.0.1" @@ -247,7 +238,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73cd58deff2140a0a8eae87e417bd01db68a33e148aa93d1e8cd837e55e312b6" dependencies = [ - "object 0.39.1", + "object", ] [[package]] @@ -508,6 +499,22 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" +[[package]] +name = "asn1-rs" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fd5ddaf0351dff5b8da21b2fb4ff8e08ddd02857f0bf69c47639106c0fff0" +dependencies = [ + "asn1-rs-derive 0.4.0", + "asn1-rs-impl 0.1.0", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 1.0.69", + "time", +] + [[package]] name = "asn1-rs" version = "0.6.2" @@ -515,7 +522,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" dependencies = [ "asn1-rs-derive 0.5.1", - "asn1-rs-impl", + "asn1-rs-impl 0.2.0", "displaydoc", "nom", "num-traits", @@ -531,7 +538,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" dependencies = [ "asn1-rs-derive 0.6.0", - "asn1-rs-impl", + "asn1-rs-impl 0.2.0", "displaydoc", "nom", "num-traits", @@ -540,6 +547,18 @@ dependencies = [ "time", ] +[[package]] +name = "asn1-rs-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "726535892e8eae7e70657b4c8ea93d26b8553afb1ce617caee529ef96d7dee6c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "synstructure 0.12.6", +] + [[package]] name = "asn1-rs-derive" version = "0.5.1" @@ -564,6 +583,17 @@ dependencies = [ "synstructure 0.13.2", ] +[[package]] +name = "asn1-rs-impl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2777730b2039ac0f95f093556e61b6d26cebed5393ca6f152717777cec3a42ed" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "asn1-rs-impl" version = "0.2.0" @@ -920,6 +950,35 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attribute-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f9ef7a68684657ce9da341c934c73b63df7a3585a61e366d84c84ed5d3d167" +dependencies = [ + "attribute-derive-macro", + "manyhow 0.14.0", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d426e5e73549125bb8e4649bdd5508f82c94ddf898fb0517246e71dfb8e98" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow 0.14.0", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 3.0.6", +] + [[package]] name = "autocfg" version = "1.5.1" @@ -974,11 +1033,23 @@ version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ - "aws-lc-sys", + "aws-lc-sys 0.45.0", "untrusted 0.7.1", "zeroize", ] +[[package]] +name = "aws-lc-sys" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] + [[package]] name = "aws-lc-sys" version = "0.45.0" @@ -1349,7 +1420,7 @@ dependencies = [ "hyper-util", "pin-project-lite", "rustls 0.21.12", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-pki-types", "tokio", @@ -1492,44 +1563,17 @@ dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", "aws-smithy-types", - "rustc_version 0.4.1", + "rustc_version", "tracing", ] -[[package]] -name = "axum" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" -dependencies = [ - "async-trait", - "axum-core 0.4.5", - "bytes", - "futures-util", - "http 1.5.0", - "http-body 1.1.0", - "http-body-util", - "itoa", - "matchit 0.7.3", - "memchr", - "mime", - "percent-encoding", - "pin-project-lite", - "rustversion", - "serde", - "sync_wrapper", - "tower 0.5.3", - "tower-layer", - "tower-service", -] - [[package]] name = "axum" version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31b698c5f9a010f6573133b09e0de5408834d0c82f8d7475a89fc1867a71cd90" dependencies = [ - "axum-core 0.5.6", + "axum-core", "axum-macros", "bytes", "form_urlencoded", @@ -1558,26 +1602,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "axum-core" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "http 1.5.0", - "http-body 1.1.0", - "http-body-util", - "mime", - "pin-project-lite", - "rustversion", - "sync_wrapper", - "tower-layer", - "tower-service", -] - [[package]] name = "axum-core" version = "0.5.6" @@ -1608,12 +1632,6 @@ dependencies = [ "syn 2.0.119", ] -[[package]] -name = "az" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be5eb007b7cacc6c660343e96f650fedf4b5a77512399eb952ca6642cf8d13f7" - [[package]] name = "backon" version = "1.6.0" @@ -1625,27 +1643,18 @@ dependencies = [ "tokio", ] -[[package]] -name = "backtrace" -version = "0.3.76" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" -dependencies = [ - "addr2line", - "cfg-if", - "libc", - "miniz_oxide 0.8.9", - "object 0.37.3", - "rustc-demangle", - "windows-link 0.2.1", -] - [[package]] name = "base16ct" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c7f02d4ea65f2c1853089ffd8d2787bdbc63de2f0d29dedbcf8ccdfa0ccd4cf" +[[package]] +name = "base16ct" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd307490d624467aa6f74b0eabb77633d1f758a7b25f12bceb0b22e08d9726f6" + [[package]] name = "base32" version = "0.5.1" @@ -1723,6 +1732,26 @@ dependencies = [ "serde", ] +[[package]] +name = "bindgen" +version = "0.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f49d8fed880d473ea71efb9bf597651e77201bdd4893efe54c9e5d65ae04ce6f" +dependencies = [ + "bitflags 2.13.2", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease 0.2.37", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex 1.3.0", + "syn 2.0.119", +] + [[package]] name = "bindgen" version = "0.71.1" @@ -2018,6 +2047,17 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "brotli" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor 4.0.3", +] + [[package]] name = "brotli" version = "7.0.0" @@ -2212,6 +2252,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "calendrical_calculations" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5abbd6eeda6885048d357edc66748eea6e0268e3dd11f326fff5bd248d779c26" +dependencies = [ + "core_maths", + "displaydoc", +] + [[package]] name = "candle-core" version = "0.9.2" @@ -2364,6 +2414,18 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "char_str" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4" +dependencies = [ + "castaway", + "get-size2", + "itoa", + "ryu", +] + [[package]] name = "chrono" version = "0.4.45" @@ -2490,11 +2552,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b77c319abfd5219629c45c34c89ba945ed3c5e49fcde9d16b6c3885f118a730" dependencies = [ "const-oid 0.9.6", - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", "x509-cert", ] +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + [[package]] name = "colorchoice" version = "1.0.5" @@ -2534,6 +2602,19 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "static_assertions", + "zmij", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2562,6 +2643,18 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-hex" +version = "1.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e59eef12462b0f9b0a3620219be5d639afd79fe39dff0a42c3997061f9298b4" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "proptest", + "serde_core", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -2700,6 +2793,21 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core_maths" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77745e017f5edba1a9c1d854f6f3a52dac8a12dd5af5d2f54aecf61e43d80d30" +dependencies = [ + "libm", +] + +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.2.17" @@ -2751,6 +2859,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "cron" version = "0.17.0" @@ -2842,6 +2956,22 @@ dependencies = [ "zeroize", ] +[[package]] +name = "crypto-bigint" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +dependencies = [ + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -2859,7 +2989,32 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ + "getrandom 0.4.3", "hybrid-array", + "rand_core 0.10.1", +] + +[[package]] +name = "cssparser" +version = "0.36.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dae61cf9c0abb83bd659dab65b7e4e38d8236824c85f0f804f173567bda257d2" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "phf 0.13.1", + "smallvec", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331" +dependencies = [ + "quote", + "syn 2.0.119", ] [[package]] @@ -2899,13 +3054,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03bb0e1cc970d482d121d9a1744999169b69a07470b3d644a7894e53fcaf4574" dependencies = [ "cmov", + "subtle", ] [[package]] name = "curl-sys" -version = "0.4.90+curl-8.21.0" +version = "0.4.91+curl-8.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97799a0d220bfb3361e0fe4936966ff8c4b24d65c3f06dfc70d7b680b44e7897" +checksum = "cd39579123e45bf762716bb893d43886ed1cabe6289455be29a4f1d4e37afb06" dependencies = [ "cc", "libc", @@ -2926,8 +3082,8 @@ dependencies = [ "cpufeatures 0.2.17", "curve25519-dalek-derive", "digest 0.10.7", - "fiat-crypto 0.2.9", - "rustc_version 0.4.1", + "fiat-crypto", + "rustc_version", "subtle", "zeroize", ] @@ -3801,7 +3957,7 @@ checksum = "c72e0409b3dbd60a5bf296cbc273a8e36bb3a3aab6abac389f1891e187d5ce14" dependencies = [ "base64 0.22.1", "capacity_builder", - "deno_error 0.7.3", + "deno_error", "deno_media_type", "deno_terminal", "dprint-swc-ext", @@ -3835,37 +3991,29 @@ dependencies = [ "url", ] -[[package]] -name = "deno_console" -version = "0.209.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66c0b8a65dcb7b38c22e5969c6454b3cb0839b7dcfb7a4f0d904de871a4c4416" -dependencies = [ - "deno_core", -] - [[package]] name = "deno_core" -version = "0.352.1" +version = "0.412.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf78f3f72ac8e09b18a588bc26a1b3c5ab853e7fb489889b2f5000654d34db5d" +checksum = "580cb2e0d797d75ff3dfc4a8f2b14aa7470f0fa31654a09a83d4cbb42b5eb9b4" dependencies = [ "anyhow", - "az", - "bincode", + "base64 0.22.1", "bit-set", "bit-vec 0.8.0", + "boxed_error", "bytes", - "capacity_builder", "cooked-waker", - "deno_core_icudata", - "deno_error 0.6.1", + "deno_error", "deno_ops", "deno_path_util", "deno_unsync", + "deno_v8", "futures", "indexmap 2.14.2", + "inventory", "libc", + "num-bigint", "parking_lot", "percent-encoding", "pin-project", @@ -3875,28 +4023,25 @@ dependencies = [ "smallvec", "sourcemap", "static_assertions", + "sys_traits", "thiserror 2.0.21", "tokio", "url", - "v8", + "uuid", "wasm_dep_analyzer", + "windows-sys 0.59.0", ] -[[package]] -name = "deno_core_icudata" -version = "0.74.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695" - [[package]] name = "deno_crypto" -version = "0.223.0" +version = "0.273.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "beb2568806b930b96fb6440ff16463befa73a36de3378986644e7134c9d52e8f" +checksum = "d9931d22328961b9a9c8d3b666a27312c1f3ddcc7ea78d8610ddfbc934ac61a0" dependencies = [ "aes 0.8.3", "aes-gcm", "aes-kw", + "argon2", "aws-lc-rs", "base64 0.22.1", "cbc", @@ -3904,12 +4049,15 @@ dependencies = [ "ctr", "curve25519-dalek", "deno_core", - "deno_error 0.6.1", - "deno_web", + "deno_crypto_provider", + "deno_error", "ecdsa", - "ed448-goldilocks", - "elliptic-curve", + "elliptic-curve 0.13.8", + "fips203", + "fips205", + "hmac 0.12.1", "num-traits", + "ocb3", "once_cell", "p256", "p384", @@ -3920,21 +4068,34 @@ dependencies = [ "serde_bytes", "sha1", "sha2 0.10.9", - "signature", - "spki", + "sha3", + "signature 2.2.0", + "spki 0.7.3", + "subtle", "thiserror 2.0.21", + "tiny-keccak", "tokio", "uuid", "x25519-dalek", ] [[package]] -name = "deno_error" -version = "0.6.1" +name = "deno_crypto_provider" +version = "0.53.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "612ec3fc481fea759141b0c57810889b0a4fb6fee8f10748677bfe492fd30486" +checksum = "fafd673e5a4eefd6d65db358b00ff60488cfa1be6515406b4371935c2e2e6baa" dependencies = [ - "deno_error_macro 0.6.1", + "aws-lc-sys 0.40.0", + "ed448-goldilocks", +] + +[[package]] +name = "deno_error" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfafd2219b29886a71aecbb3449e462deed1b2c474dc5b12f855f0e58c478931" +dependencies = [ + "deno_error_macro", "libc", "serde", "serde_json", @@ -3942,32 +4103,11 @@ dependencies = [ "url", ] -[[package]] -name = "deno_error" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3007d3f1ea92ea503324ae15883aac0c2de2b8cf6fead62203ff6a67161007ab" -dependencies = [ - "deno_error_macro 0.7.3", - "libc", -] - [[package]] name = "deno_error_macro" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8380a4224d5d2c3f84da4d764c4326cac62e9a1e3d4960442d29136fc07be863" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "deno_error_macro" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b565e60a9685cdf312c888665b5f8647ac692a7da7e058a5e2268a466da8eaf" +checksum = "1c28ede88783f14cd8aae46ca89f230c226b40e4a81ab06fa52ed72af84beb2f" dependencies = [ "proc-macro2", "quote", @@ -3976,31 +4116,30 @@ dependencies = [ [[package]] name = "deno_features" -version = "0.6.0" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "487773bb24b92f3b88d1c9ef0d4d15888641a2e2ea4d4cdfd6fda12cae26317c" +checksum = "2fef16a40daeb19acfc5345aa5bb367a36ecffef18b7ad1a594f0f01fe7c20c5" dependencies = [ "deno_core", - "serde", - "serde_json", ] [[package]] name = "deno_fetch" -version = "0.233.0" +version = "0.283.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cbe158bec955790105ded69978b1245b97c65092f7bf184add344c0e78efffa" +checksum = "20e792f76d45e20cf6f1ad46157bbe1a0a895a6f09ec4d99b0911112be8e05e3" dependencies = [ + "async-compression", "base64 0.22.1", "bytes", "data-url", "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_fs", + "deno_io", "deno_path_util", "deno_permissions", "deno_tls", - "dyn-clone", "error_reporter", "h2 0.4.20", "hickory-resolver", @@ -4011,8 +4150,7 @@ dependencies = [ "hyper-util", "ipnet", "percent-encoding", - "rustls-webpki 0.102.8", - "serde", + "rustls-webpki 0.103.15", "serde_json", "thiserror 2.0.21", "tokio", @@ -4021,62 +4159,69 @@ dependencies = [ "tokio-util", "tokio-vsock", "tower 0.5.3", - "tower-http", "tower-service", ] [[package]] name = "deno_fs" -version = "0.119.0" +version = "0.169.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca83a95cea7bcdf19dac1888ef6f04aff69d251640a4d085bc6089c06b4b181" +checksum = "a0b0ae3c55f4af9d33113ba4935828a2e9a7bfcbf4d3058533403b9ef332a966" dependencies = [ "async-trait", - "base32", "boxed_error", "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_io", + "deno_maybe_sync", "deno_path_util", "deno_permissions", "filetime", "junction", "libc", - "nix 0.27.1", + "nix 0.30.1", "rand 0.8.5", "rayon", + "same-file", "serde", "thiserror 2.0.21", - "winapi", + "tokio", "windows-sys 0.59.0", ] [[package]] name = "deno_io" -version = "0.119.0" +version = "0.169.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597aac25be261a1bd545d6f47a80439c60e600c076f9847b3718a592883b2a73" +checksum = "ba495b6c141d34e49fb2dbe7ca746d7ac068b1962470776eb341c59059b758f6" dependencies = [ "async-trait", "deno_core", - "deno_error 0.6.1", + "deno_error", + "deno_permissions", "deno_subprocess_windows", "filetime", - "fs3", "libc", "log", - "nix 0.27.1", - "once_cell", + "nix 0.30.1", "os_pipe", "parking_lot", "pin-project", "rand 0.8.5", "tokio", "uuid", - "winapi", "windows-sys 0.59.0", ] +[[package]] +name = "deno_maybe_sync" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57e5e72b38015b5e478b8d739dfa818d563329e03a72d6ce5cfb2ecbcd7f8d41" +dependencies = [ + "rustc-hash 2.1.3", +] + [[package]] name = "deno_media_type" version = "0.3.3" @@ -4103,17 +4248,21 @@ dependencies = [ [[package]] name = "deno_net" -version = "0.201.0" +version = "0.251.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb88f1ea2762065d6cc0316372acc22696ab6004943aa029f80546f9faf1b491" +checksum = "19ba02152d978f30e2eb5f8d074b343363f53cb1364b5c304b3176da916a06c0" dependencies = [ "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_features", "deno_permissions", + "deno_signals", "deno_tls", + "deno_tunnel", "hickory-proto", "hickory-resolver", + "libc", + "log", "pin-project", "quinn", "rustls-tokio-stream", @@ -4125,32 +4274,33 @@ dependencies = [ "tokio-vsock", "url", "web-transport-proto", + "x509-parser 0.15.1", ] [[package]] name = "deno_ops" -version = "0.228.1" +version = "0.288.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf8dbe5abf37d270bb853c5dfe45fbe3b1b6c453877cc11d7fe84e9862a6dbc" +checksum = "977748624aad80164dbe385c8c907bc0041c2e6849f92528e362e03132c64c52" dependencies = [ "indexmap 2.14.2", - "proc-macro-rules", "proc-macro2", "quote", "stringcase", "strum", "strum_macros", "syn 2.0.119", + "syn-match", "thiserror 2.0.21", ] [[package]] name = "deno_path_util" -version = "0.4.0" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "516f813389095889776b81cc9108ff6f336fd9409b4b12fc0138aea23d2708e1" +checksum = "78c7e98943f0d068928906db0c7bde89de684fa32c6a8018caacc4cee2cdd72b" dependencies = [ - "deno_error 0.6.1", + "deno_error", "percent-encoding", "sys_traits", "thiserror 2.0.21", @@ -4159,77 +4309,100 @@ dependencies = [ [[package]] name = "deno_permissions" -version = "0.68.0" +version = "0.118.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baa14d9c3fbba59836ffbaac7f736a2f48d4a5fc209a2103d1e1b898915232a5" +checksum = "f8e9d37215f52e29b5ad9d9065c476168ffb888cd2ad93b0ae153f22304851a0" dependencies = [ "capacity_builder", - "deno_error 0.6.1", + "chrono", + "deno_error", "deno_path_util", "deno_terminal", "deno_unsync", + "dunce", "fqdn", - "ipnetwork", + "ipnet", "libc", "log", - "nix 0.27.1", + "nix 0.30.1", "once_cell", "parking_lot", - "percent-encoding", "serde", "serde_json", "sys_traits", - "temp_deno_which", "thiserror 2.0.21", + "unicode-normalization", "url", - "winapi", + "which 8.0.6", + "windows-sys 0.59.0", +] + +[[package]] +name = "deno_signals" +version = "0.50.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038ed4b09a6d36c81ac436c4d10076e9e01cc67faf93d701c541887f8eeddf36" +dependencies = [ + "libc", + "signal-hook", + "thiserror 2.0.21", + "tokio", "windows-sys 0.59.0", ] [[package]] name = "deno_subprocess_windows" -version = "0.4.0" +version = "0.54.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bc6b10059f0ccb14c6e0319c5275f0407fb2f9ffe405cd555700561999ea4bf" +checksum = "04d4f451b2f94d2847de4895d39ca5aa264b90af34e72fd79ddc1d0da05a5ea0" dependencies = [ - "fastrand 2.5.0", "futures-channel", "libc", + "rand 0.8.5", "windows-sys 0.59.0", ] [[package]] name = "deno_telemetry" -version = "0.31.0" +version = "0.81.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "377581966bd34e85ce230f7f4b1dee7e16c6bf7f845b4690648507ce7ba105d9" +checksum = "8831dc236ed1c66a90c8dfddab93ad44fbb28950fc476c78306b9100233ff694" dependencies = [ "async-trait", "deno_core", - "deno_error 0.6.1", + "deno_error", + "deno_net", + "deno_signals", + "deno_terminal", "deno_tls", + "flate2", "http-body-util", "hyper 1.12.0", "hyper-rustls 0.27.10", "hyper-util", "log", "once_cell", - "opentelemetry 0.27.1", - "opentelemetry-http 0.27.0", - "opentelemetry-otlp 0.27.0", - "opentelemetry-semantic-conventions 0.27.0", - "opentelemetry_sdk 0.27.1", + "opentelemetry 0.32.0", + "opentelemetry-http 0.32.0", + "opentelemetry-otlp 0.32.0", + "opentelemetry-proto 0.32.0", + "opentelemetry-semantic-conventions 0.32.1", + "opentelemetry_sdk 0.32.1", "pin-project", + "prost 0.14.4", "serde", + "sys_traits", "thiserror 2.0.21", "tokio", + "tokio-vsock", + "tower-service", ] [[package]] name = "deno_terminal" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f71c27009e0141dedd315f1dfa3ebb0a6ca4acce7c080fac576ea415a465f6" +checksum = "f3ba8041ae7319b3ca6a64c399df4112badcbbe0868b4517637647614bede4be" dependencies = [ "once_cell", "termcolor", @@ -4237,23 +4410,40 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.196.0" +version = "0.246.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7835eb6a8d114703b0293573fbb8885bc2c81e8c35363b9e89391dd3c541fa6" +checksum = "6229f7a00441b2acbf7fae58dd14bded0e835f212653402083430826e640f99d" dependencies = [ "deno_core", - "deno_error 0.6.1", + "deno_error", "deno_native_certs", - "rustls 0.23.35", + "log", + "rustls 0.23.40", "rustls-pemfile 2.2.0", "rustls-tokio-stream", - "rustls-webpki 0.102.8", + "rustls-webpki 0.103.15", "serde", "thiserror 2.0.21", "tokio", "webpki-roots 0.26.11", ] +[[package]] +name = "deno_tunnel" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e432929e0c167f2a003b963ee406c714fcd947a87d4f9edc7b3255e87be9881d" +dependencies = [ + "pin-project", + "quinn", + "serde", + "serde_json", + "socket2 0.6.5", + "thiserror 2.0.21", + "tokio", + "tracing", +] + [[package]] name = "deno_unsync" version = "0.4.4" @@ -4266,42 +4456,43 @@ dependencies = [ ] [[package]] -name = "deno_url" -version = "0.209.0" +name = "deno_v8" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cccbb10fb59f29b04161055a65f7fe83364f9e8317d4d2ce3bd7979faf63a87" +checksum = "85cdff08dedf79da6a40f97554f47d59d419ba3df8061b8b31789cd2cbe00433" dependencies = [ - "deno_core", - "deno_error 0.6.1", - "urlpattern", + "v8", + "v8x", ] [[package]] name = "deno_web" -version = "0.240.0" +version = "0.290.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d6b8a97cc90b6aaea20fe200cfa7e5b6953dd33ecda0af50d6d360387c33df4" +checksum = "7510967ba4ec695959c1a518748c3b564fd9ca7a8f3eb13bd397707a3ca232da" dependencies = [ "async-trait", - "base64-simd", + "brotli 6.0.0", "bytes", + "cssparser", "deno_core", - "deno_error 0.6.1", - "deno_permissions", + "deno_error", "encoding_rs", "flate2", "futures", "serde", "thiserror 2.0.21", "tokio", + "unicode-normalization", + "urlpattern", "uuid", ] [[package]] name = "deno_webidl" -version = "0.209.0" +version = "0.259.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20bbfac0cf15918f7cbd0b55c366ac20ad65c825f362fa684c3104984254d3b" +checksum = "40af42313723a8d3114c4e5b007abd1cab4400a0b68929794c556b2e792409c7" dependencies = [ "deno_core", ] @@ -4319,6 +4510,30 @@ dependencies = [ "zeroize", ] +[[package]] +name = "der" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" +dependencies = [ + "const-oid 0.10.2", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "8.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbd676fbbab537128ef0278adb5576cf363cff6aa22a7b24effe97347cfab61e" +dependencies = [ + "asn1-rs 0.5.2", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "der-parser" version = "9.0.0" @@ -4367,6 +4582,25 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive-io" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58cc7f4740088458993d183a200fe56e62378736f94bf0e2dd45807407e7bb94" +dependencies = [ + "derive-io-macros", + "tokio", +] + +[[package]] +name = "derive-io-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d0a1bd7eeab72097740967d03d53db5fbaf8e3c0dd471ebdefa43ce445a20a6" +dependencies = [ + "proc-macro2", +] + [[package]] name = "derive_builder" version = "0.12.0" @@ -4456,7 +4690,6 @@ dependencies = [ "proc-macro2", "quote", "syn 2.0.119", - "unicode-xid", ] [[package]] @@ -4467,7 +4700,7 @@ checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" dependencies = [ "proc-macro2", "quote", - "rustc_version 0.4.1", + "rustc_version", "syn 2.0.119", ] @@ -4524,6 +4757,38 @@ dependencies = [ "ctutils", ] +[[package]] +name = "diplomat" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8b37a6cea72eb6df5143d3e091d4c34cb0cf4046dd1ddcc082988886fe1f809" +dependencies = [ + "diplomat_core", + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "diplomat-runtime" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae517853d9f82b2d144393b1c32c34c46c2e361de2b6c44a8fdfce23beb211cd" + +[[package]] +name = "diplomat_core" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b18e806f9ecb92df828d080efe47984ae1193958295d4a84fb8b32c9a1aa1b" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "smallvec", + "strck", + "syn 3.0.6", +] + [[package]] name = "dirs" version = "4.0.0" @@ -4681,6 +4946,27 @@ dependencies = [ "text_lines", ] +[[package]] +name = "drop_bomb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" + +[[package]] +name = "dtoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c3cf4824e2d5f025c7b531afcb2325364084a16806f6d47fbc1f5fbd9960590" + +[[package]] +name = "dtoa-short" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87" +dependencies = [ + "dtoa", +] + [[package]] name = "duct" version = "0.13.7" @@ -4727,12 +5013,12 @@ version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der", + "der 0.7.10", "digest 0.10.7", - "elliptic-curve", + "elliptic-curve 0.13.8", "rfc6979", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", ] [[package]] @@ -4741,8 +5027,8 @@ version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "115531babc129696a58c64a4fef0a8bf9e9698629fb97e9e40767d235cfbcd53" dependencies = [ - "pkcs8", - "signature", + "pkcs8 0.10.2", + "signature 2.2.0", ] [[package]] @@ -4756,21 +5042,35 @@ dependencies = [ "rand_core 0.6.4", "serde", "sha2 0.10.9", - "signature", + "signature 2.2.0", "subtle", "zeroize", ] [[package]] -name = "ed448-goldilocks" -version = "0.8.3" +name = "ed448" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06924531e9e90130842b012e447f85bdaf9161bc8a0f8092be8cb70b01ebe092" +checksum = "ae112a25f86ae3598d4e8533ed1e65149cec6eb21918e7a6f4c06dddec370263" dependencies = [ - "fiat-crypto 0.1.20", - "hex", + "pkcs8 0.11.0", + "signature 3.0.0", +] + +[[package]] +name = "ed448-goldilocks" +version = "0.14.0-pre.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b805154de2e68f59874ec217ca36790dcffe500cd872c60fe509d28d0814a74d" +dependencies = [ + "ed448", + "elliptic-curve 0.14.1", + "hash2curve", + "rand_core 0.10.1", + "serdect 0.4.3", + "shake", + "signature 3.0.0", "subtle", - "zeroize", ] [[package]] @@ -4800,20 +5100,39 @@ version = "0.13.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" dependencies = [ - "base16ct", + "base16ct 0.2.0", "base64ct", - "crypto-bigint", + "crypto-bigint 0.5.5", "digest 0.10.7", - "ff", + "ff 0.13.1", "generic-array", - "group", + "group 0.13.0", "hkdf", "pem-rfc7468", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "sec1", + "sec1 0.7.3", "serde_json", - "serdect", + "serdect 0.2.0", + "subtle", + "zeroize", +] + +[[package]] +name = "elliptic-curve" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d65aa39b3a5c1c9c1b745c9a019234bb7a21b77abcb4f4d266d706e2d577d65" +dependencies = [ + "base16ct 1.0.0", + "crypto-bigint 0.7.5", + "crypto-common 0.2.2", + "ff 0.14.0", + "group 0.14.0", + "hybrid-array", + "pkcs8 0.11.0", + "rand_core 0.10.1", + "sec1 0.8.1", "subtle", "zeroize", ] @@ -5012,6 +5331,18 @@ dependencies = [ "regex-syntax 0.8.11", ] +[[package]] +name = "fastbloom" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" +dependencies = [ + "foldhash 0.2.0", + "libm", + "portable-atomic", + "siphasher 1.0.4", +] + [[package]] name = "fastdivide" version = "0.4.2" @@ -5044,10 +5375,14 @@ dependencies = [ ] [[package]] -name = "fiat-crypto" -version = "0.1.20" +name = "ff" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e825f6987101665dea6ec934c09ec6d721de7bc1bf92248e1d5810c8cd636b77" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" +dependencies = [ + "rand_core 0.10.1", + "subtle", +] [[package]] name = "fiat-crypto" @@ -5071,6 +5406,30 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" +[[package]] +name = "fips203" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8bdb6454f692ca2a2b45cd554c6828c639d7f9c968cf83a678899ec4443a280" +dependencies = [ + "rand_core 0.6.4", + "sha3", + "subtle", + "zeroize", +] + +[[package]] +name = "fips205" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f5626bf5534df4ebdbd2536465d7eaa8a9dc2cdeb7e036e0ecf291dcc80ffb6" +dependencies = [ + "rand_core 0.6.4", + "sha2 0.10.9", + "sha3", + "zeroize", +] + [[package]] name = "fixedbitset" version = "0.5.7" @@ -5090,7 +5449,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ "bitflags 2.13.2", - "rustc_version 0.4.1", + "rustc_version", ] [[package]] @@ -5173,9 +5532,9 @@ dependencies = [ [[package]] name = "fqdn" -version = "0.3.12" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb540cf7bc4fe6df9d8f7f0c974cfd0dce8ed4e9e8884e73433b503ee78b4e7d" +checksum = "886ac788f62d16d6b0f26b2fa762b34ef16ebfb4b624c2c15fbcadc9173c0f72" [[package]] name = "from_variant" @@ -5193,17 +5552,6 @@ version = "1.20260821.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28c70238056ad1dbfdf035222a571d2e9baf5b6a4c308cfa87bc8cbe7f2f3840" -[[package]] -name = "fs3" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb17cf6ed704f72485332f6ab65257460c4f9f3083934cf402bf9f5b3b600a90" -dependencies = [ - "libc", - "rustc_version 0.2.3", - "winapi", -] - [[package]] name = "fs4" version = "0.13.1" @@ -5533,6 +5881,31 @@ dependencies = [ "zeroize", ] +[[package]] +name = "get-size-derive2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28f0daa179069133df60523e0f59d0a6bd2d3db9ccae01b630896d269c4a802e" +dependencies = [ + "attribute-derive", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "get-size2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050a5d07359ebfac7f0b74916d94cfa30575e45fdc7e9bfbb0f76d5d4203325a" +dependencies = [ + "compact_str 0.10.0", + "get-size-derive2", + "hashbrown 0.17.1", + "ordermap", + "smallvec", + "thin-vec", +] + [[package]] name = "gethostname" version = "0.4.3" @@ -5614,12 +5987,6 @@ dependencies = [ "polyval", ] -[[package]] -name = "gimli" -version = "0.32.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" - [[package]] name = "git-version" version = "0.3.9" @@ -5721,7 +6088,7 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "886aa8ec755382a1fdf4651f6e6ec01f2f3bf49f2cb0f068b9a74cafd574a715" dependencies = [ - "prost", + "prost 0.13.5", "prost-types", "tonic 0.12.3", ] @@ -5783,11 +6150,22 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" dependencies = [ - "ff", + "ff 0.13.1", "rand_core 0.6.4", "subtle", ] +[[package]] +name = "group" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" +dependencies = [ + "ff 0.14.0", + "rand_core 0.10.1", + "subtle", +] + [[package]] name = "gzip-header" version = "1.1.0" @@ -5850,6 +6228,16 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "hash2curve" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1eaf40612d7d854743e7189228a6d528f0f6e8502cf6a0cb831d28a218b7f3f6" +dependencies = [ + "digest 0.11.3", + "elliptic-curve 0.14.1", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -5997,11 +6385,10 @@ dependencies = [ [[package]] name = "hickory-proto" -version = "0.25.0-alpha.5" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d00147af6310f4392a31680db52a3ed45a2e0f68eb18e8c3fe5537ecc96d9e2" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" dependencies = [ - "async-recursion", "async-trait", "cfg-if", "data-encoding", @@ -6013,6 +6400,7 @@ dependencies = [ "ipnet", "once_cell", "rand 0.9.5", + "ring 0.17.14", "serde", "thiserror 2.0.21", "tinyvec", @@ -6023,9 +6411,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.25.0-alpha.5" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5762f69ebdbd4ddb2e975cd24690bf21fe6b2604039189c26acddbc427f12887" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" dependencies = [ "cfg-if", "futures-util", @@ -6217,7 +6605,9 @@ version = "0.4.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" dependencies = [ + "subtle", "typenum", + "zeroize", ] [[package]] @@ -6344,7 +6734,7 @@ dependencies = [ "hyper 1.12.0", "hyper-util", "log", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "tokio", "tokio-rustls 0.26.6", @@ -6461,6 +6851,28 @@ dependencies = [ "cc", ] +[[package]] +name = "icu_calendar" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58655df2f728e46e4eee80bddebefacf52ec3e77cdb925014b47c5a5905eb55c" +dependencies = [ + "calendrical_calculations", + "displaydoc", + "icu_calendar_data", + "icu_locale_core", + "icu_locale_fallback", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_calendar_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc00caaa3fb3201ff7a18aa458e8c3ab042b9da154cfdf948bdde3936c31c36e" + [[package]] name = "icu_collections" version = "2.3.0" @@ -6683,6 +7095,12 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + [[package]] name = "inventory" version = "0.3.25" @@ -6703,17 +7121,6 @@ dependencies = [ "windows-sys 0.48.0", ] -[[package]] -name = "io-uring" -version = "0.7.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed3bd0ecfbb87805f538bb7b32e5239ca0763890c623e349860ecba69469f2bb" -dependencies = [ - "bitflags 2.13.2", - "cfg-if", - "libc", -] - [[package]] name = "ipconfig" version = "0.3.4" @@ -6733,15 +7140,6 @@ version = "2.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "791930b43c0d5973160d90a8f3894509f2b273430f5c5c73b668636d0287c5c0" -[[package]] -name = "ipnetwork" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf466541e9d546596ee94f9f69590f89473455f88372423e0008fc1a7daf100e" -dependencies = [ - "serde", -] - [[package]] name = "is-macro" version = "0.3.8" @@ -6802,12 +7200,27 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "ixdtf" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d3667095d64c3ecffc96463a21157b04bf3e252f6e8d5750b20c02e33c194e3" + [[package]] name = "jiff" version = "0.2.38" @@ -6887,7 +7300,7 @@ checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" dependencies = [ "proc-macro2", "quote", - "rustc_version 0.4.1", + "rustc_version", "simd_cesu8", "syn 2.0.119", ] @@ -6995,7 +7408,7 @@ dependencies = [ "pem", "serde", "serde_json", - "signature", + "signature 2.2.0", "simple_asn1", "zeroize", ] @@ -7022,6 +7435,25 @@ dependencies = [ "serde_json", ] +[[package]] +name = "keccak" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures 0.2.17", +] + +[[package]] +name = "keccak" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8f198d1db720e4940b5a493201d199d9f24f568f8f746bd13706243a2f71598" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.1", +] + [[package]] name = "keyed_priority_queue" version = "0.4.2" @@ -7093,7 +7525,7 @@ dependencies = [ "k8s-openapi", "kube-core", "pem", - "rustls 0.23.35", + "rustls 0.23.40", "secrecy", "serde", "serde_json", @@ -7166,12 +7598,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "lalrpop-util" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" - [[package]] name = "lapin" version = "2.5.5" @@ -7582,7 +8008,7 @@ dependencies = [ "base64 0.22.1", "gethostname", "mail-builder", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "smtp-proto", "tokio", @@ -7590,76 +8016,30 @@ dependencies = [ "webpki-roots 0.26.11", ] -[[package]] -name = "malachite" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" -dependencies = [ - "malachite-base", - "malachite-nz", - "malachite-q", -] - -[[package]] -name = "malachite-base" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" -dependencies = [ - "hashbrown 0.14.5", - "itertools 0.11.0", - "libm", - "ryu", -] - -[[package]] -name = "malachite-bigint" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" -dependencies = [ - "derive_more 1.0.0", - "malachite", - "num-integer", - "num-traits", - "paste", -] - -[[package]] -name = "malachite-nz" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7" -dependencies = [ - "itertools 0.11.0", - "libm", - "malachite-base", -] - -[[package]] -name = "malachite-q" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191" -dependencies = [ - "itertools 0.11.0", - "malachite-base", - "malachite-nz", -] - [[package]] name = "manyhow" version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" dependencies = [ - "manyhow-macros", + "manyhow-macros 0.11.4", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "manyhow" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c69ea6e469ca785c846b25dd69eac9e9c916b51d8ada1f212866787ee1da12d" +dependencies = [ + "manyhow-macros 0.14.0", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "manyhow-macros" version = "0.11.4" @@ -7671,6 +8051,17 @@ dependencies = [ "quote", ] +[[package]] +name = "manyhow-macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc5c1b2c4000056dae3619fea36e79ce79dabfee4907d2dcad6612b189e370" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + [[package]] name = "mappable-rc" version = "0.1.1" @@ -7938,7 +8329,7 @@ checksum = "a4db8a44120571277accfaa3f3d91e7d3989d601d817c2fc01a9391b86135666" dependencies = [ "darling 0.23.0", "heck", - "manyhow", + "manyhow 0.11.4", "num-bigint", "proc-macro-crate", "proc-macro2", @@ -8072,6 +8463,7 @@ dependencies = [ "cfg-if", "cfg_aliases", "libc", + "memoffset", ] [[package]] @@ -8291,7 +8683,6 @@ checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" dependencies = [ "num-integer", "num-traits", - "rand 0.8.5", "serde", ] @@ -8435,15 +8826,6 @@ dependencies = [ "url", ] -[[package]] -name = "object" -version = "0.37.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" -dependencies = [ - "memchr", -] - [[package]] name = "object" version = "0.39.1" @@ -8490,6 +8872,18 @@ dependencies = [ "web-time", ] +[[package]] +name = "ocb3" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c196e0276c471c843dd5777e7543a36a298a4be942a2a688d8111cd43390dedb" +dependencies = [ + "aead", + "cipher 0.4.4", + "ctr", + "subtle", +] + [[package]] name = "odpic-sys" version = "0.1.1" @@ -8499,6 +8893,15 @@ dependencies = [ "cc", ] +[[package]] +name = "oid-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bedf36ffb6ba96c2eb7144ef6270557b52e54b20c0a8e1eb2ff99a6c6959bff" +dependencies = [ + "asn1-rs 0.5.2", +] + [[package]] name = "oid-registry" version = "0.7.1" @@ -8528,6 +8931,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -8667,23 +9074,23 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.27.1" +version = "0.30.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" +checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6" dependencies = [ "futures-core", "futures-sink", "js-sys", "pin-project-lite", - "thiserror 1.0.69", + "thiserror 2.0.21", "tracing", ] [[package]] name = "opentelemetry" -version = "0.30.0" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" dependencies = [ "futures-core", "futures-sink", @@ -8705,18 +9112,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "opentelemetry-http" -version = "0.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a8a7f5f6ba7c1b286c2fbca0454eaba116f63bbe69ed250b642d36fbb04d80" -dependencies = [ - "async-trait", - "bytes", - "http 1.5.0", - "opentelemetry 0.27.1", -] - [[package]] name = "opentelemetry-http" version = "0.30.0" @@ -8731,24 +9126,15 @@ dependencies = [ ] [[package]] -name = "opentelemetry-otlp" -version = "0.27.0" +name = "opentelemetry-http" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +checksum = "5683015d09e2df236ef005b17f6f196f0d5f6313c4fa43a7b6a53b52776e4331" dependencies = [ "async-trait", - "futures-core", + "bytes", "http 1.5.0", - "opentelemetry 0.27.1", - "opentelemetry-http 0.27.0", - "opentelemetry-proto 0.27.0", - "opentelemetry_sdk 0.27.1", - "prost", - "serde_json", - "thiserror 1.0.69", - "tokio", - "tonic 0.12.3", - "tracing", + "opentelemetry 0.32.0", ] [[package]] @@ -8762,7 +9148,7 @@ dependencies = [ "opentelemetry-http 0.30.0", "opentelemetry-proto 0.30.0", "opentelemetry_sdk 0.30.0", - "prost", + "prost 0.13.5", "reqwest 0.12.28", "thiserror 2.0.21", "tokio", @@ -8771,17 +9157,19 @@ dependencies = [ ] [[package]] -name = "opentelemetry-proto" -version = "0.27.0" +name = "opentelemetry-otlp" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +checksum = "9966929966d17620d7c316c643ba62631826e10021409357772d5eea84f62c35" dependencies = [ - "hex", - "opentelemetry 0.27.1", - "opentelemetry_sdk 0.27.1", - "prost", - "serde", - "tonic 0.12.3", + "http 1.5.0", + "opentelemetry 0.32.0", + "opentelemetry-http 0.32.0", + "opentelemetry-proto 0.32.0", + "opentelemetry_sdk 0.32.1", + "prost 0.14.4", + "serde_json", + "thiserror 2.0.21", ] [[package]] @@ -8794,16 +9182,24 @@ dependencies = [ "hex", "opentelemetry 0.30.0", "opentelemetry_sdk 0.30.0", - "prost", + "prost 0.13.5", "serde", "tonic 0.13.1", ] [[package]] -name = "opentelemetry-semantic-conventions" -version = "0.27.0" +name = "opentelemetry-proto" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" +checksum = "56d658ba1faf63f7b9c492cfbe6e0ec365440a16132d3270c1065f7b33f1b638" +dependencies = [ + "base64 0.22.1", + "const-hex", + "opentelemetry 0.32.0", + "opentelemetry_sdk 0.32.1", + "prost 0.14.4", + "serde", +] [[package]] name = "opentelemetry-semantic-conventions" @@ -8812,25 +9208,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83d059a296a47436748557a353c5e6c5705b9470ef6c95cfc52c21a8814ddac2" [[package]] -name = "opentelemetry_sdk" -version = "0.27.1" +name = "opentelemetry-semantic-conventions" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" -dependencies = [ - "async-trait", - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "opentelemetry 0.27.1", - "percent-encoding", - "rand 0.8.5", - "serde_json", - "thiserror 1.0.69", - "tokio", - "tokio-stream", - "tracing", -] +checksum = "c913ac17a6c451661ee255f4625d143e51647ae78ebd969b75e41c4442f4fe47" [[package]] name = "opentelemetry_sdk" @@ -8850,6 +9231,24 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "opentelemetry_sdk" +version = "0.32.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b59f80e1ac4d5ff7a2db8fb6c80badb7f0f3f858211fba08dd9aaec750894f9" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry 0.32.0", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror 2.0.21", + "tokio", + "tokio-stream", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -8901,6 +9300,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordermap" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +dependencies = [ + "indexmap 2.14.2", +] + [[package]] name = "os_pipe" version = "1.2.1" @@ -8939,7 +9347,7 @@ checksum = "3cae83056e7cb770211494a0ecf66d9fa7eba7d00977e5bb91f0e925b40b937f" dependencies = [ "cbc", "cms", - "der", + "der 0.7.10", "des 0.8.1", "hex", "hmac 0.12.1", @@ -8960,7 +9368,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" dependencies = [ "ecdsa", - "elliptic-curve", + "elliptic-curve 0.13.8", "primeorder", "sha2 0.10.9", ] @@ -8972,7 +9380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe42f1670a52a47d448f14b6a5c61dd78fce51856e68edaa38f7ae3a46b8d6b6" dependencies = [ "ecdsa", - "elliptic-curve", + "elliptic-curve 0.13.8", "primeorder", "sha2 0.10.9", ] @@ -8983,9 +9391,9 @@ version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fc9e2161f1f215afdfce23677034ae137bbd45016a880c2eb3ba8eb95f085b2" dependencies = [ - "base16ct", + "base16ct 0.2.0", "ecdsa", - "elliptic-curve", + "elliptic-curve 0.13.8", "primeorder", "rand_core 0.6.4", "sha2 0.10.9", @@ -9224,7 +9632,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ - "phf_macros", + "phf_macros 0.11.3", "phf_shared 0.11.3", ] @@ -9243,6 +9651,7 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ + "phf_macros 0.13.1", "phf_shared 0.13.1", "serde", ] @@ -9253,7 +9662,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator", + "phf_generator 0.11.3", "phf_shared 0.11.3", ] @@ -9267,19 +9676,42 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand 2.5.0", + "phf_shared 0.13.1", +] + [[package]] name = "phf_macros" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" dependencies = [ - "phf_generator", + "phf_generator 0.11.3", "phf_shared 0.11.3", "proc-macro2", "quote", "syn 2.0.119", ] +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "phf_shared" version = "0.11.3" @@ -9380,9 +9812,9 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8ffb9f10fa047879315e6625af03c164b16962a5368d724ed16323b68ace47f" dependencies = [ - "der", - "pkcs8", - "spki", + "der 0.7.10", + "pkcs8 0.10.2", + "spki 0.7.3", ] [[package]] @@ -9393,9 +9825,9 @@ checksum = "695b3df3d3cc1015f12d70235e35b6b79befc5fa7a9b95b951eab1dd07c9efc2" dependencies = [ "cms", "const-oid 0.9.6", - "der", + "der 0.7.10", "digest 0.10.7", - "spki", + "spki 0.7.3", "x509-cert", "zeroize", ] @@ -9408,11 +9840,11 @@ checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" dependencies = [ "aes 0.8.3", "cbc", - "der", + "der 0.7.10", "pbkdf2", "scrypt", "sha2 0.10.9", - "spki", + "spki 0.7.3", ] [[package]] @@ -9421,8 +9853,18 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der 0.8.2", + "spki 0.8.1", ] [[package]] @@ -9608,7 +10050,7 @@ version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" dependencies = [ - "elliptic-curve", + "elliptic-curve 0.13.8", ] [[package]] @@ -9617,7 +10059,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit 0.25.16+spec-1.1.0", + "toml_edit 0.25.17+spec-1.1.0", ] [[package]] @@ -9643,29 +10085,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "proc-macro-rules" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07c277e4e643ef00c1233393c673f655e3672cf7eb3ba08a00bdd0ea59139b5f" -dependencies = [ - "proc-macro-rules-macros", - "proc-macro2", - "syn 2.0.119", -] - -[[package]] -name = "proc-macro-rules-macros" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "207fffb0fe655d1d47f6af98cc2793405e85929bdbc420d685554ff07be27ac7" -dependencies = [ - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "proc-macro-utils" version = "0.10.0" @@ -9739,6 +10158,21 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bitflags 2.13.2", + "num-traits", + "rand 0.9.5", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax 0.8.11", + "unarray", +] + [[package]] name = "prost" version = "0.13.5" @@ -9746,7 +10180,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.5", +] + +[[package]] +name = "prost" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "528ac67416ff8646872a3c02cad9cc4ee5dc9f9540c9b10771855c95cb2e5ae1" +dependencies = [ + "bytes", + "prost-derive 0.14.4", ] [[package]] @@ -9762,13 +10206,26 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "prost-derive" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" +dependencies = [ + "anyhow", + "itertools 0.14.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "prost-types" version = "0.13.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" dependencies = [ - "prost", + "prost 0.13.5", ] [[package]] @@ -9886,7 +10343,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.3", - "rustls 0.23.35", + "rustls 0.23.40", "socket2 0.6.5", "thiserror 2.0.21", "tokio", @@ -9902,13 +10359,14 @@ checksum = "0e750cca55fe4f0439a15d0bb529da9651e79993e8e72c61a899a36d462befbe" dependencies = [ "aws-lc-rs", "bytes", + "fastbloom", "getrandom 0.4.3", "lru-slab", "rand 0.10.3", "rand_pcg", "ring 0.17.14", "rustc-hash 2.1.3", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "slab", "thiserror 2.0.21", @@ -9940,6 +10398,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48356e493823dc3c1d7e9fc7b3ff204ecec48afbdd718fc4134091b88eb63f3d" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf1c5de6a7cc231c71e03c77aa4c7019097859ea7029156b54d2894a8430704" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 3.0.6", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -10094,6 +10574,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -10373,7 +10862,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-pki-types", "serde", @@ -10420,7 +10909,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "rustls-platform-verifier", "serde", @@ -10476,6 +10965,16 @@ dependencies = [ "wasmtimer", ] +[[package]] +name = "resb" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22d392791f3c6802a1905a509e9d1a6039cbbcb5e9e00e5a6d3661f7c874f390" +dependencies = [ + "potential_utf", + "serde_core", +] + [[package]] name = "resolv-conf" version = "0.7.6" @@ -10666,14 +11165,114 @@ dependencies = [ "num-integer", "num-traits", "pkcs1", - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", - "spki", + "signature 2.2.0", + "spki 0.7.3", "subtle", "zeroize", ] +[[package]] +name = "ruff_python_ast" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15434b06df7d930fad2818639a4e0fec4af9c770121c45a96b72da7feaae9ead" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags 2.13.2", + "char_str", + "compact_str 0.10.0", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "thin-vec", + "thiserror 2.0.21", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a22811d752aa9e255d7dd72524d14352cf9bc654ef79b399bf1eeeb6640fbb" +dependencies = [ + "bitflags 2.13.2", + "bstr", + "drop_bomb", + "get-size2", + "hashbrown 0.17.1", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "ruff_unicode_names2", + "rustc-hash 2.1.3", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0322a5059fe0feb2e010ca6f31c24ac289f6b7e9e38359539491b10c57f1c5bc" +dependencies = [ + "itertools 0.15.0", + "memchr", + "ruff_source_file", + "ruff_text_size", + "rustc-hash 2.1.3", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ae809a289b5c791f82aa45df33f2a92323e7a1c1d265c10e9ac884027a2e43" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf6344d6915ced9316312c1ee3f24c4300c54295bc4c3b8f4e95b633ae5443b" +dependencies = [ + "get-size2", +] + +[[package]] +name = "ruff_unicode_names2" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13418b8fd029d597e35dedc36e91f47122d26075e4c86557574d9441753fb375" +dependencies = [ + "phf 0.11.3", + "ruff_unicode_names2_generator", +] + +[[package]] +name = "ruff_unicode_names2_generator" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fd82af49d5685d9054e68185987acaaf62542f671b6df8935155664876bc39" +dependencies = [ + "phf_codegen", + "rand 0.8.5", +] + [[package]] name = "rumqttc" version = "0.24.0" @@ -10747,12 +11346,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "rustc-demangle" -version = "0.1.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b74b56ffa8bb2830709a538c2cbcae9aa062db0d2a42563bfb09bdaae44020eb" - [[package]] name = "rustc-hash" version = "1.1.0" @@ -10765,22 +11358,13 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" -[[package]] -name = "rustc_version" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" -dependencies = [ - "semver 0.9.0", -] - [[package]] name = "rustc_version" version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" dependencies = [ - "semver 1.0.28", + "semver", ] [[package]] @@ -10860,9 +11444,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.35" +version = "0.23.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" dependencies = [ "aws-lc-rs", "log", @@ -10881,7 +11465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70cc376c6ba1823ae229bacf8ad93c136d93524eab0e4e5e0e4f96b9c4e5b212" dependencies = [ "log", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.7.3", "rustls-pki-types", "rustls-webpki 0.103.15", @@ -10963,7 +11547,7 @@ dependencies = [ "jni", "log", "once_cell", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-platform-verifier-android", "rustls-webpki 0.103.15", @@ -10981,12 +11565,13 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-tokio-stream" -version = "0.5.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa7dc7c991d9164e55bbf1558029eb5b84d32cc4d61a7df5b8641b2deedc4b3" +checksum = "0560d12c0d8c672f849197de91b9ee61f5bf9aa024c97aaeeb112ec2f6c347fd" dependencies = [ + "derive-io", "futures", - "rustls 0.23.35", + "rustls 0.23.40", "socket2 0.5.10", "tokio", ] @@ -11024,63 +11609,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "rustpython-ast" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5" -dependencies = [ - "is-macro", - "malachite-bigint", - "rustpython-parser-core", - "static_assertions", -] - -[[package]] -name = "rustpython-parser" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb" -dependencies = [ - "anyhow", - "is-macro", - "itertools 0.11.0", - "lalrpop-util", - "log", - "malachite-bigint", - "num-traits", - "phf 0.11.3", - "phf_codegen", - "rustc-hash 1.1.0", - "rustpython-ast", - "rustpython-parser-core", - "tiny-keccak", - "unic-emoji-char", - "unic-ucd-ident", - "unicode_names2", -] - -[[package]] -name = "rustpython-parser-core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad" -dependencies = [ - "is-macro", - "memchr", - "rustpython-parser-vendored", -] - -[[package]] -name = "rustpython-parser-vendored" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6" -dependencies = [ - "memchr", - "once_cell", -] - [[package]] name = "rustversion" version = "1.0.23" @@ -11288,11 +11816,25 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ - "base16ct", - "der", + "base16ct 0.2.0", + "der 0.7.10", "generic-array", - "pkcs8", - "serdect", + "pkcs8 0.10.2", + "serdect 0.2.0", + "subtle", + "zeroize", +] + +[[package]] +name = "sec1" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" +dependencies = [ + "base16ct 1.0.0", + "ctutils", + "der 0.8.2", + "hybrid-array", "subtle", "zeroize", ] @@ -11342,27 +11884,12 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" -dependencies = [ - "semver-parser", -] - [[package]] name = "semver" version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" -[[package]] -name = "semver-parser" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" - [[package]] name = "seq-macro" version = "0.3.6" @@ -11541,16 +12068,16 @@ dependencies = [ [[package]] name = "serde_v8" -version = "0.261.1" +version = "0.321.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3495190857461e87a2716141043218aad5281f219f54a03b7ebbe605b3b931df" +checksum = "fd3b6f6140192f569b88997e2ea7afeef6aa38b64374e84784ea1d1ea35679d3" dependencies = [ - "deno_error 0.6.1", + "deno_error", + "deno_v8", "num-bigint", "serde", "smallvec", "thiserror 2.0.21", - "v8", ] [[package]] @@ -11620,7 +12147,17 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a84f14a19e9a014bb9f4512488d9829a68e04ecabffb0f9904cd1ace94598177" dependencies = [ - "base16ct", + "base16ct 0.2.0", + "serde", +] + +[[package]] +name = "serdect" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "66cf8fedced2fcf12406bcb34223dffb92eaf34908ede12fed414c82b7f00b3e" +dependencies = [ + "base16ct 1.0.0", "serde", ] @@ -11695,6 +12232,27 @@ dependencies = [ "digest 0.11.3", ] +[[package]] +name = "sha3" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" +dependencies = [ + "digest 0.10.7", + "keccak 0.1.6", +] + +[[package]] +name = "shake" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09057cb2149ad4cbd2da1e26b351f9a4c354219421229c69c3063e6f61947c4a" +dependencies = [ + "digest 0.11.3", + "keccak 0.2.2", + "sponge-cursor", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -11735,6 +12293,16 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -11751,9 +12319,9 @@ version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1e303f8205714074f6068773f0e29527e0453937fe837c9717d066635b65f31" dependencies = [ - "pkcs8", + "pkcs8 0.10.2", "rand_core 0.6.4", - "signature", + "signature 2.2.0", "zeroize", ] @@ -11767,6 +12335,16 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "signature" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" +dependencies = [ + "digest 0.11.3", + "rand_core 0.10.1", +] + [[package]] name = "simd-adler32" version = "0.3.10" @@ -11779,7 +12357,7 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" dependencies = [ - "rustc_version 0.4.1", + "rustc_version", "simdutf8", ] @@ -11947,7 +12525,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der", + "der 0.7.10", +] + +[[package]] +name = "spki" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef958a98b9d5da290cfc78946e9f3e61e1e62a18db0d92cac0b83cc161491a9" +dependencies = [ + "base64ct", + "der 0.8.2", ] [[package]] @@ -11962,6 +12550,12 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "sponge-cursor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" + [[package]] name = "sql-builder" version = "3.1.1" @@ -12041,7 +12635,7 @@ dependencies = [ "memchr", "once_cell", "percent-encoding", - "rustls 0.23.35", + "rustls 0.23.40", "serde", "serde_json", "sha2 0.10.9", @@ -12238,6 +12832,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "strck" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42316e70da376f3d113a68d138a60d8a9883c604fe97942721ec2068dab13a9f" +dependencies = [ + "unicode-ident", +] + [[package]] name = "string_enum" version = "1.0.2" @@ -12438,7 +13041,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bcf55c2d7555c93f4945e29f93b7529562be97ba16e60dd94c25724d746174ac" dependencies = [ "ascii", - "compact_str", + "compact_str 0.7.1", "memchr", "num-bigint", "once_cell", @@ -12751,6 +13354,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn-match" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b8f0a9004d6aafa6a588602a1119e6cdaacec9921aa1605383e6e7d6258fd6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -12760,6 +13374,18 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-xid", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -12793,11 +13419,14 @@ dependencies = [ [[package]] name = "sys_traits" -version = "0.1.16" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc4707edf3196e8037ee45018d1bb1bfb233b0e4fc440fa3d3f25bc69bfdaf26" +checksum = "c7a35f11f9773f11930a7d1e6276766f1577400ee2530b7f0cf3844c2dcbf12b" dependencies = [ + "junction", + "libc", "sys_traits_macros", + "windows-sys 0.59.0", ] [[package]] @@ -13052,15 +13681,6 @@ dependencies = [ "rustls-pemfile 2.2.0", ] -[[package]] -name = "temp_deno_which" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "366c5ccd670145885feb6efd6bbf2478ed236c4c3839046fcc8e2a1a84c51091" -dependencies = [ - "either", -] - [[package]] name = "tempfile" version = "3.27.0" @@ -13074,6 +13694,40 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "temporal_capi" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f48eb9ca4ca22caaa48d1e7a534ad3fd598a23d1a71945e8ad5ac0be501fa5f2" +dependencies = [ + "diplomat", + "diplomat-runtime", + "icu_calendar", + "icu_locale_core", + "num-traits", + "temporal_rs", + "timezone_provider", + "writeable", + "zoneinfo64", +] + +[[package]] +name = "temporal_rs" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a978fef907a27736827d336c0bfaab14fb8a7509dd6dc2b43c44e51d59aca5b" +dependencies = [ + "calendrical_calculations", + "core_maths", + "icu_calendar", + "icu_locale_core", + "ixdtf", + "num-traits", + "timezone_provider", + "tinystr", + "writeable", +] + [[package]] name = "termcolor" version = "1.4.1" @@ -13112,6 +13766,12 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" @@ -13274,6 +13934,18 @@ dependencies = [ "time-core", ] +[[package]] +name = "timezone_provider" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ec4f4eebb4817fde02a411aedc7ac5f66c89c68c49e5d4b17a8139f077af52" +dependencies = [ + "tinystr", + "zerotrie", + "zerovec", + "zoneinfo64", +] + [[package]] name = "tiny-keccak" version = "2.0.2" @@ -13350,23 +14022,20 @@ dependencies = [ [[package]] name = "tokio" -version = "1.46.1" +version = "1.53.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "e95f91fcc7a621e8b030f6aa23c71fe9838ae2fb4d8118b75602a328f5144044" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", - "socket2 0.5.10", + "socket2 0.6.5", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -13395,13 +14064,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] @@ -13490,7 +14159,7 @@ version = "0.26.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c9cc2678c2cdd569ef8215e2afd7954ada2ae20b4fdd2c5fe6139a3b02d105db" dependencies = [ - "rustls 0.23.35", + "rustls 0.23.40", "tokio", ] @@ -13644,9 +14313,9 @@ dependencies = [ [[package]] name = "toml_edit" -version = "0.25.16+spec-1.1.0" +version = "0.25.17+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88f38f3af8bf71841c67a990f4e2b52143b2d0a1a994c72513ff4c14977fc722" +checksum = "e3641d5bbb5349a79e1020a242d251efbc546ad8048d133958323ce9c40a9c9c" dependencies = [ "indexmap 2.14.2", "toml_datetime 1.1.2+spec-1.1.0", @@ -13656,9 +14325,9 @@ dependencies = [ [[package]] name = "toml_parser" -version = "1.1.4+spec-1.1.0" +version = "1.1.5+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bed810ee007d8e7f9cf20b4774e7198142fa27186e0f805871da7f3c39eaf333" +checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c" dependencies = [ "winnow 1.0.4", ] @@ -13669,13 +14338,10 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" dependencies = [ - "async-stream", "async-trait", - "axum 0.7.9", "base64 0.22.1", "bytes", "flate2", - "h2 0.4.20", "http 1.5.0", "http-body 1.1.0", "http-body-util", @@ -13684,9 +14350,8 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "rustls-pemfile 2.2.0", - "socket2 0.5.10", "tokio", "tokio-rustls 0.26.6", "tokio-stream", @@ -13704,7 +14369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9" dependencies = [ "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "flate2", @@ -13717,7 +14382,7 @@ dependencies = [ "hyper-util", "percent-encoding", "pin-project", - "prost", + "prost 0.13.5", "rustls-native-certs 0.8.4", "socket2 0.5.10", "tokio", @@ -13775,7 +14440,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "151b5a3e3c45df17466454bb74e9ecedecc955269bdedbf4d150dfa393b55a36" dependencies = [ - "axum-core 0.5.6", + "axum-core", "cookie", "futures-util", "http 1.5.0", @@ -13973,9 +14638,9 @@ dependencies = [ [[package]] name = "tree-sitter-language" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca0d1bf6fdd806e43ae5198f82f527056d359def39e54e67a0f478ac09dac081" +checksum = "d0af592be68c579aa78a16846bd19422978c3c52e438523d45ff5d1bff1f9d4a" [[package]] name = "tree-sitter-r" @@ -14129,6 +14794,12 @@ dependencies = [ "web-time", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unic-char-property" version = "0.9.0" @@ -14150,17 +14821,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-emoji-char" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-category" version = "0.9.0" @@ -14173,17 +14833,6 @@ dependencies = [ "unic-ucd-version", ] -[[package]] -name = "unic-ucd-ident" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-version" version = "0.9.0" @@ -14271,28 +14920,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "unicode_names2" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" -dependencies = [ - "phf 0.11.3", - "unicode_names2_generator", -] - -[[package]] -name = "unicode_names2_generator" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" -dependencies = [ - "getopts", - "log", - "phf_codegen", - "rand 0.8.5", -] - [[package]] name = "universal-hash" version = "0.5.1" @@ -14344,7 +14971,7 @@ dependencies = [ "log", "native-tls", "once_cell", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-pki-types", "serde", "serde_json", @@ -14374,13 +15001,13 @@ checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" [[package]] name = "urlpattern" -version = "0.3.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70acd30e3aa1450bc2eece896ce2ad0d178e9c079493819301573dae3c37ba6d" +checksum = "0f805818f843b548bacc19609eb3619dd2850e54746f5cada37927393c2ef4ec" dependencies = [ + "icu_properties", "regex", "serde", - "unic-ucd-ident", "url", ] @@ -14428,20 +15055,36 @@ dependencies = [ [[package]] name = "v8" -version = "137.3.0" +version = "150.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33995a1fee055ff743281cde33a41f0d618ee0bdbe8bdf6859e11864499c2595" +checksum = "42a978ff11f15b24e5c05a7123cf2b68f41e763546699781a924ef4e2cf43a49" dependencies = [ - "bindgen 0.71.1", + "bindgen 0.72.1", "bitflags 2.13.2", "fslock", "gzip-header", "home", "miniz_oxide 0.8.9", "paste", + "temporal_capi", "which 6.0.3", ] +[[package]] +name = "v8x" +version = "149.4.0-rc.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3701ffb3692a24808503d17f758b0a3f688d2c451779247982566b5941033e63" +dependencies = [ + "bindgen 0.70.1", + "bitflags 2.13.2", + "cc", + "paste", + "temporal_capi", + "temporal_rs", + "timezone_provider", +] + [[package]] name = "valuable" version = "0.1.1" @@ -14661,11 +15304,11 @@ dependencies = [ [[package]] name = "wasm_dep_analyzer" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51cf5f08b357e64cd7642ab4bbeb11aecab9e15520692129624fb9908b8df2c" +checksum = "a10e6b67c951a84de7029487e0e0a496860dae49f6699edd279d5ff35b8fbf54" dependencies = [ - "deno_error 0.6.1", + "deno_error", "thiserror 2.0.21", ] @@ -14766,6 +15409,12 @@ dependencies = [ "winsafe", ] +[[package]] +name = "which" +version = "8.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f" + [[package]] name = "whoami" version = "1.6.1" @@ -14829,14 +15478,14 @@ dependencies = [ [[package]] name = "windmill" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-nats", "aws-config", "aws-credential-types", "aws-sdk-sqs", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "constant_time_eq 0.3.1", @@ -14856,7 +15505,7 @@ dependencies = [ "rdkafka", "reqwest 0.13.5", "rumqttc", - "rustls 0.23.35", + "rustls 0.23.40", "serde", "serde_derive", "serde_json", @@ -14918,7 +15567,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.828.0" +version = "1.830.0" dependencies = [ "async-stream", "async-trait", @@ -14935,6 +15584,7 @@ dependencies = [ "indexmap 2.14.2", "lazy_static", "mime_guess", + "rand 0.9.5", "reqwest 0.13.5", "serde", "serde_json", @@ -14952,9 +15602,9 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -14965,7 +15615,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "argon2", @@ -14976,7 +15626,7 @@ dependencies = [ "async_zip", "aws-sdk-config", "aws-sigv4", - "axum 0.8.9", + "axum", "base32", "base64 0.22.1", "bytes", @@ -15018,7 +15668,7 @@ dependencies = [ "reqwest 0.13.5", "rsa", "rust-embed", - "rustls 0.23.35", + "rustls 0.23.40", "samael", "serde", "serde_json", @@ -15105,9 +15755,9 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", "hyper 1.12.0", @@ -15128,9 +15778,9 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -15145,10 +15795,10 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", "itertools 0.14.0", @@ -15171,7 +15821,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.828.0" +version = "1.830.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15181,9 +15831,9 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "itertools 0.14.0", "serde", @@ -15198,9 +15848,9 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "ed25519-dalek", @@ -15220,10 +15870,10 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "candle-core", "candle-nn", "candle-transformers", @@ -15243,9 +15893,9 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "sql-builder", @@ -15259,9 +15909,9 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "hyper 1.12.0", "serde", @@ -15281,9 +15931,9 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "globset", "lazy_static", @@ -15303,9 +15953,9 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -15317,14 +15967,14 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-nats", "aws-config", "aws-credential-types", "aws-sdk-sqs", - "axum 0.8.9", + "axum", "base64 0.22.1", "futures", "hex", @@ -15353,10 +16003,10 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "http 1.5.0", @@ -15379,9 +16029,9 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "flate2", "futures", @@ -15389,7 +16039,7 @@ dependencies = [ "lazy_static", "quick_cache", "reqwest 0.13.5", - "semver 1.0.28", + "semver", "serde", "serde_json", "sha2 0.10.9", @@ -15407,10 +16057,10 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "http 1.5.0", "indexmap 2.14.2", "itertools 0.14.0", @@ -15429,9 +16079,9 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "chrono-tz", "serde", @@ -15449,10 +16099,10 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "chrono", "futures", "http 1.5.0", @@ -15491,10 +16141,10 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "chrono", @@ -15520,7 +16170,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.828.0" +version = "1.830.0" dependencies = [ "lazy_static", "serde", @@ -15532,10 +16182,10 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.828.0" +version = "1.830.0" dependencies = [ "argon2", - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", "hyper 1.12.0", @@ -15556,9 +16206,9 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "serde", "serde_json", @@ -15570,9 +16220,9 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.828.0" +version = "1.830.0" dependencies = [ - "axum 0.8.9", + "axum", "chrono", "constant_time_eq 0.3.1", "futures", @@ -15609,7 +16259,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.828.0" +version = "1.830.0" dependencies = [ "chrono", "lazy_static", @@ -15624,10 +16274,10 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", - "axum 0.8.9", + "axum", "k8s-openapi", "kube", "serde", @@ -15643,7 +16293,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.828.0" +version = "1.830.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15657,7 +16307,7 @@ dependencies = [ "aws-sdk-secretsmanager", "aws-sdk-sts", "aws-smithy-types-convert", - "axum 0.8.9", + "axum", "backon", "base64 0.22.1", "bitflags 2.13.2", @@ -15711,18 +16361,18 @@ dependencies = [ "reqwest-middleware", "reqwest-retry", "rsa", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "ryu-js", "schemars 0.8.22", - "semver 1.0.28", + "semver", "serde", "serde_json", "serde_yml", "sha1", "sha2 0.10.9", "size", - "spki", + "spki 0.7.3", "sqlx", "strum", "strum_macros", @@ -15757,7 +16407,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.828.0" +version = "1.830.0" dependencies = [ "chrono", "futures", @@ -15777,7 +16427,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.828.0" +version = "1.830.0" dependencies = [ "regex", "serde", @@ -15794,7 +16444,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15821,7 +16471,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "futures", @@ -15838,7 +16488,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.828.0" +version = "1.830.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15854,7 +16504,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", @@ -15875,11 +16525,11 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "backon", "base64 0.22.1", "chrono", @@ -15906,12 +16556,12 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "arc-swap", "async-oauth2", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "hex", @@ -15931,7 +16581,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-stream", @@ -15940,7 +16590,7 @@ dependencies = [ "aws-credential-types", "aws-sdk-sts", "aws-smithy-types-convert", - "axum 0.8.9", + "axum", "bytes", "chrono", "datafusion", @@ -15967,7 +16617,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "futures", @@ -15985,7 +16635,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.828.0" +version = "1.830.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15994,7 +16644,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16006,7 +16656,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -16018,7 +16668,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "gosyn", @@ -16030,7 +16680,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16042,7 +16692,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -16054,7 +16704,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "nu-parser", @@ -16065,7 +16715,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16076,30 +16726,30 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde_json", "windmill-parser", ] [[package]] name = "windmill-parser-py-asset" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "windmill-parser", "windmill-parser-sql-asset", ] [[package]] name = "windmill-parser-py-imports" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", @@ -16109,7 +16759,8 @@ dependencies = [ "phf 0.11.3", "regex", "regex-lite", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde", "serde_json", "sqlx", @@ -16121,7 +16772,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -16133,7 +16784,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16147,7 +16798,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16164,7 +16815,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16177,7 +16828,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "serde", @@ -16189,7 +16840,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16207,7 +16858,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16223,11 +16874,12 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "serde", "serde_json", "sha2 0.10.9", @@ -16239,7 +16891,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -16253,11 +16905,11 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", - "axum 0.8.9", + "axum", "backon", "chrono", "chrono-tz", @@ -16292,23 +16944,19 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "const_format", "deno_ast", - "deno_console", "deno_core", "deno_crypto", - "deno_error 0.6.1", + "deno_error", "deno_fetch", - "deno_fs", - "deno_io", "deno_net", "deno_permissions", "deno_telemetry", "deno_tls", - "deno_url", "deno_web", "deno_webidl", "futures", @@ -16317,13 +16965,15 @@ dependencies = [ "rcgen", "regex", "reqwest 0.13.5", - "rustls 0.23.35", + "rustls 0.23.40", "serde", "serde_json", "sqlx", + "sys_traits", "tokio", "tracing", "uuid", + "v8", "winapi", "windmill-common", "windmill-parser-ts", @@ -16332,7 +16982,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.828.0" +version = "1.830.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16343,11 +16993,11 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "dashmap", @@ -16378,11 +17028,11 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "chrono", "futures", "serde", @@ -16402,11 +17052,11 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "chrono", "http 1.5.0", "hyper 1.12.0", @@ -16435,11 +17085,11 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "futures", "itertools 0.14.0", @@ -16462,11 +17112,11 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "chrono", @@ -16495,11 +17145,11 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "lazy_static", "regex", @@ -16515,11 +17165,11 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "chrono", @@ -16549,11 +17199,11 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "chrono", "constant_time_eq 0.3.1", @@ -16586,11 +17236,11 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "itertools 0.14.0", "rdkafka", @@ -16609,11 +17259,11 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "bytes", "itertools 0.14.0", @@ -16633,12 +17283,12 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-nats", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "itertools 0.14.0", "nkeys", @@ -16657,11 +17307,11 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "byteorder", "bytes", "chrono", @@ -16692,7 +17342,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", @@ -16701,7 +17351,7 @@ dependencies = [ "aws-sdk-sqs", "aws-sdk-sts", "aws-smithy-types", - "axum 0.8.9", + "axum", "backon", "chrono", "itertools 0.14.0", @@ -16720,11 +17370,11 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-trait", - "axum 0.8.9", + "axum", "base64 0.22.1", "futures", "http 1.5.0", @@ -16745,7 +17395,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -16764,14 +17414,14 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-once-cell", "async-recursion", "async-stream", "async-trait", - "axum 0.8.9", + "axum", "backon", "base64 0.22.1", "bit-vec 0.6.3", @@ -16816,7 +17466,7 @@ dependencies = [ "postgres-protocol", "process-wrap", "prometheus", - "prost", + "prost 0.13.5", "quick_cache", "rand 0.9.5", "rcgen", @@ -16825,7 +17475,7 @@ dependencies = [ "reqwest-middleware", "rsa", "rust_decimal", - "rustls 0.23.35", + "rustls 0.23.40", "rustls-native-certs 0.8.4", "rustls-pemfile 2.2.0", "serde", @@ -16883,7 +17533,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.828.0" +version = "1.830.0" dependencies = [ "bytes", "futures", @@ -17479,8 +18129,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" dependencies = [ "const-oid 0.9.6", - "der", - "spki", + "der 0.7.10", + "spki 0.7.3", +] + +[[package]] +name = "x509-parser" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7069fba5b66b9193bd2c5d3d4ff12b839118f6bcbef5328efafafb5395cf63da" +dependencies = [ + "asn1-rs 0.5.2", + "data-encoding", + "der-parser 8.2.0", + "lazy_static", + "nom", + "oid-registry 0.6.1", + "rusticata-macros", + "thiserror 1.0.69", + "time", ] [[package]] @@ -17592,18 +18259,18 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.61" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "879fb705ce98c32e41ebdb970fbe1204f8492423b314c6ab0354c3e7b5542866" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.61" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "708882a28301d604fa039cc7727607a98d04c4b86dc76ec9cc683f805d709759" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" dependencies = [ "proc-macro2", "quote", @@ -17710,6 +18377,19 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" +[[package]] +name = "zoneinfo64" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed6eb2607e906160c457fd573e9297e65029669906b9ac8fb1b5cd5e055f0705" +dependencies = [ + "calendrical_calculations", + "icu_locale_core", + "potential_utf", + "resb", + "serde", +] + [[package]] name = "zstd" version = "0.13.3" @@ -17730,9 +18410,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.1.0+zstd.1.5.7" +version = "2.1.1+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +checksum = "aeec9eaf2dffbbd09201e23bd0ffcbaa33bb8e9266a10734fd7ed90a85eca078" dependencies = [ "cc", "pkg-config", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index e0b97275b2..d627328e27 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.828.0" +version = "1.830.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.828.0" +version = "1.830.0" authors = ["Ruben Fiszel "] edition = "2021" @@ -451,7 +451,7 @@ headers = "^0" hyper = { version = "^1", features = ["full"] } hyper-tls = "^0.6" hyper-util = { version = "^0.1", features = ["client-legacy", "http1", "tokio"] } -tokio = { version = "=1.46.1", features = ["full", "tracing", "time"] } +tokio = { version = "=1.53.2", features = ["full", "tracing", "time"] } tokio-stream = { version = "0.1.17" } tower = "^0" tower-http = { version = "^0.6", features = ["trace", "cors", "catch-panic"] } @@ -488,9 +488,11 @@ arc-swap = "1" ed25519-dalek = { version = "2", features = ["rand_core"] } magic-crypt = "^3" git-version = "^0" -malachite = "=0.4.18" -malachite-bigint = "=0.2.0" -rustpython-parser = "^0" +# Ruff's parser crates are internal to ruff and carry no semver guarantee: pin them exactly, +# and to the same release. +ruff_python_parser = "=0.0.17" +ruff_python_ast = "=0.0.17" +ruff_text_size = "=0.0.17" pep440_rs = "0.7.3" php-parser-rs = { git = "https://github.com/php-rust-tools/parser", rev = "ec4cb411dec09450946ef57920b7ffced7f6495d" } cron = "^0" @@ -519,33 +521,29 @@ aws-sdk-rds = "^1" async-trait = "0.1.88" -v8 = "=137.1.0" # Exact version NOTE: Do not forget to update version and hash in flake.nix -# deno_* pin set: deno v2.4.0 base, with deno_ast force-overridden to =0.51.0. -# Rationale: deno_ast 0.51.0 is the first version pulling swc_common =14.0.4, -# the first swc_common patch that dropped `pub use serde::__private as serde;` -# (the line that capped our workspace serde pin at =1.0.220). v2.4.0's other -# pins keep deno_tls at 0.196.0 which uses permissive `rustls ^0.23.11`, -# compatible with aws-sdk-bedrockruntime's `^0.23.31` requirement. deno_tls -# 0.198+ tightened that to exact `=0.23.28`, which would have made any -# meaningful deno bump resolver-impossible against aws-sdk. -deno_fetch = "0.233.0" -deno_tls = "0.196.0" -deno_console = "0.209.0" -deno_url = "0.209.0" -deno_webidl = "0.209.0" -deno_web = "0.240.0" -# Sibling release of the pinned deno_core 0.352 / deno_web 0.240 stack -# (its deps are deno_core ^0.352, deno_web ^0.240, deno_error =0.6.1). A newer -# deno_crypto would force bumping the whole deno stack. -deno_crypto = "0.223.0" -deno_io = "0.119.0" -deno_fs = "0.119.0" -deno_net = "0.201.0" -deno_core = "0.352.0" +# deno_* pin set: the crates deno published together on 2026-09-16 (deno_core +# 0.412). +# V8 is pinned exactly because flake.nix fetches the prebuilt archive of that +# version: update its version and hash there with any bump. No default features, +# so the pin leaves the feature set deno_core picks untouched. +v8 = { version = "=150.4.0", default-features = false } +# These release as a matched set: each pins its siblings, deno_error exactly, +# and deno_tls pins rustls exactly (=0.23.40, the workspace `rustls` pin below +# must equal it). A bump moves all of them at once. +# deno_ast is independent of that set and stays at 0.51.0: a newer one pulls +# swc_common 17, which would force bumping every swc_* pin the parsers use. +deno_fetch = "0.283.0" +deno_tls = "0.246.0" +deno_webidl = "0.259.0" +deno_web = "0.290.0" +deno_crypto = "0.273.0" +deno_net = "0.251.0" +deno_core = "0.412.0" deno_ast = { version = "=0.51.0", features = ["transpiling"] } -deno_permissions = "0.68.0" -deno_telemetry = "0.31.0" -deno_error = "=0.6.1" +deno_permissions = "0.118.0" +deno_telemetry = "0.81.0" +deno_error = "=0.7.1" +sys_traits = { version = "0.1.28", features = ["real", "libc", "winapi"] } rustls-pemfile = "2.2.0" # only used with special deno_core_mac feature to prevent ffi issue on macos, requires libffi to be installed @@ -682,7 +680,7 @@ aws-sdk-sqs = "^1.89.0" aws-sdk-sts = "^1.91.0" aws-sdk-sso = "=1.77.0" aws-sdk-ssooidc = "=1.78.0" -rustls = "=0.23.35" +rustls = "=0.23.40" async-once-cell = "0.5.4" size = "0.5.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 14bd926cf5..012f205d9c 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -127575148c84bb6132d2946c0a95c6dea0f58bb2 +06610ecc950ed64a2709d7f1aa8c7b49a95fdbae diff --git a/backend/migrations/20261008165042_runnable_job_stats.down.sql b/backend/migrations/20261008165042_runnable_job_stats.down.sql new file mode 100644 index 0000000000..21208736b4 --- /dev/null +++ b/backend/migrations/20261008165042_runnable_job_stats.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS runnable_job_stats; diff --git a/backend/migrations/20261008165042_runnable_job_stats.up.sql b/backend/migrations/20261008165042_runnable_job_stats.up.sql new file mode 100644 index 0000000000..cf676a037e --- /dev/null +++ b/backend/migrations/20261008165042_runnable_job_stats.up.sql @@ -0,0 +1,17 @@ +-- Hourly resource rollup per runnable and worker group, fed by the workers. +-- Memory figures are in kB, like v2_job_completed.memory_peak. +-- No foreign key to workspace: every worker upserts here, and the key would cost each row +-- a lookup and a lock on its workspace. Workspace deletion removes the rows itself. +CREATE TABLE IF NOT EXISTS runnable_job_stats ( + hour BIGINT NOT NULL, + workspace_id VARCHAR(50) NOT NULL, + runnable_path VARCHAR(255) NOT NULL, + worker_group TEXT NOT NULL, + job_count INTEGER NOT NULL DEFAULT 0, + total_duration_ms BIGINT NOT NULL DEFAULT 0, + max_memory_peak INTEGER NOT NULL DEFAULT 0, + sum_memory_peak BIGINT NOT NULL DEFAULT 0, + memory_sample_count INTEGER NOT NULL DEFAULT 0, + total_cpu_ms BIGINT NOT NULL DEFAULT 0, + PRIMARY KEY (hour, workspace_id, runnable_path, worker_group) +); diff --git a/backend/parsers/windmill-parser-py-asset/Cargo.toml b/backend/parsers/windmill-parser-py-asset/Cargo.toml index ddd6446079..84562dcf0e 100644 --- a/backend/parsers/windmill-parser-py-asset/Cargo.toml +++ b/backend/parsers/windmill-parser-py-asset/Cargo.toml @@ -11,6 +11,6 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true windmill-parser-sql-asset.workspace = true -rustpython-parser.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true anyhow.workspace = true diff --git a/backend/parsers/windmill-parser-py-asset/src/lib.rs b/backend/parsers/windmill-parser-py-asset/src/lib.rs index 9c70b93d64..2919b74607 100644 --- a/backend/parsers/windmill-parser-py-asset/src/lib.rs +++ b/backend/parsers/windmill-parser-py-asset/src/lib.rs @@ -1,5 +1,8 @@ -use rustpython_ast::{Constant, Expr, ExprConstant, Visitor}; -use rustpython_parser::{ast::Suite, Parse}; +use ruff_python_ast::{ + visitor::{walk_expr, walk_stmt, Visitor}, + Expr, ExprCall, ExprDict, FStringPartRef, InterpolatedStringElement, Keyword, Stmt, StmtAssign, +}; +use ruff_python_parser::parse_module; use std::collections::HashMap; use windmill_parser::asset_parser::{ asset_was_used, merge_assets, parse_asset_syntax, parse_pipeline_annotations, AssetKind, @@ -8,12 +11,12 @@ use windmill_parser::asset_parser::{ use AssetUsageAccessType::*; pub fn parse_assets(input: &str) -> anyhow::Result { - let ast = Suite::parse(input, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(input) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); let mut assets_finder = AssetsFinder { assets: vec![], var_identifiers: HashMap::new() }; - ast.into_iter() - .for_each(|stmt| assets_finder.visit_stmt(stmt)); + assets_finder.visit_body(&ast); for (kind, path, _) in assets_finder.var_identifiers.into_values() { // if a db = wmill.datatable() was never used (e.g db.query(...)), @@ -43,13 +46,72 @@ struct AssetsFinder { var_identifiers: HashMap, } -impl Visitor for AssetsFinder { +impl<'a> Visitor<'a> for AssetsFinder { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + if let Stmt::Assign(node) = stmt { + self.track_assignment(node); + } + // Continue with generic visit to catch any other assets in the statement + walk_stmt(self, stmt); + } + + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + // A detected asset call does not recurse, + // so this is only reached when no further context was found + Expr::StringLiteral(s) => self.visit_str(s.value.to_str()), + Expr::FString(f) => { + // Literal text between two interpolations reads as one string, across the + // pieces of an implicitly concatenated f-string too. + let mut literal = String::new(); + for part in &f.value { + match part { + FStringPartRef::Literal(s) => literal.push_str(&s.value), + FStringPartRef::FString(f) => { + for element in &f.elements { + match element { + InterpolatedStringElement::Literal(l) => { + literal.push_str(&l.value) + } + InterpolatedStringElement::Interpolation(_) => { + self.visit_str(&std::mem::take(&mut literal)) + } + } + } + } + } + } + self.visit_str(&literal); + walk_expr(self, expr); + } + Expr::Call(node) => { + if self.visit_expr_call_inner(node).is_err() { + walk_expr(self, expr); + } + } + _ => walk_expr(self, expr), + } + } +} + +impl AssetsFinder { + fn visit_str(&mut self, s: &str) { + if let Some((kind, path)) = parse_asset_syntax(s, false) { + self.assets.push(ParseAssetsResult { + kind, + path: path.to_string(), + access_type: None, + columns: None, + }); + } + } + // Handle assignment statements like: x = wmill.datatable('name') - fn visit_stmt_assign(&mut self, node: rustpython_ast::StmtAssign) { + fn track_assignment(&mut self, node: &StmtAssign) { // Check if the value is a call to a tracked function if let Some(Expr::Name(expr_name)) = node.targets.first() { // Remove any tracked variables with that name in case of reassignment - let Ok(var_name) = expr_name.id.parse::(); + let var_name = expr_name.id.to_string(); let removed = self.var_identifiers.remove(&var_name); // if a db = wmill.datatable() or similar was removed, but never used (e.g db.query(...)), // we still want to register the asset as unknown access type @@ -69,59 +131,12 @@ impl Visitor for AssetsFinder { if let Some((kind, name, schema)) = self.extract_asset_from_call(&node.value) { // Track target variable - let Ok(var_name) = expr_name.id.parse::(); self.var_identifiers .insert(var_name, (kind.clone(), name.clone(), schema.clone())); } } - // Continue with generic visit to catch any other assets in the expression - self.generic_visit_stmt_assign(node); } - // visit_call_expr will not recurse if it detects an asset, - // so this will only be called when no further context was found - fn visit_expr_constant(&mut self, node: ExprConstant) { - match node.value { - Constant::Str(s) => { - if let Some((kind, path)) = parse_asset_syntax(&s, false) { - self.assets.push(ParseAssetsResult { - kind, - path: path.to_string(), - access_type: None, - columns: None, - }); - } - } - _ => self.generic_visit_expr_constant(node), - } - } - - fn visit_expr_call(&mut self, node: rustpython_ast::ExprCall) { - match self.visit_expr_call_inner(&node) { - Ok(_) => {} - Err(_) => { - // Check keyword arguments for assets before falling back to generic visit - for keyword in &node.keywords { - if let Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) = - &keyword.value - { - if let Some((kind, path)) = parse_asset_syntax(s, false) { - self.assets.push(ParseAssetsResult { - kind, - path: path.to_string(), - access_type: None, - columns: None, - }); - } - } - } - self.generic_visit_expr_call(node); - } - } - } -} - -impl AssetsFinder { /// Extract asset info from calls like wmill.datatable('name'), wmill.ducklake('name'), etc. fn extract_asset_from_call( &self, @@ -147,18 +162,20 @@ impl AssetsFinder { // Get the first argument (the asset name) let name = call + .arguments .args .first() .or_else(|| { // Try keyword argument 'name' or 'path' - call.keywords + call.arguments + .keywords .iter() .find(|kw| matches!(kw.arg.as_deref(), Some("name") | Some("path"))) .map(|kw| &kw.value) }) .and_then(|first_arg| { - if let Expr::Constant(ExprConstant { value: Constant::Str(name), .. }) = first_arg { - Some(name.clone()) + if let Expr::StringLiteral(name) = first_arg { + Some(name.value.to_str().to_string()) } else { None } @@ -179,44 +196,39 @@ impl AssetsFinder { Some((kind, name, schema)) } - fn visit_expr_call_inner(&mut self, node: &rustpython_ast::ExprCall) -> Result<(), ()> { + fn visit_expr_call_inner(&mut self, node: &ExprCall) -> Result<(), ()> { let ident: String = node .func .as_name_expr() - .and_then(|o| o.id.parse().ok()) + .map(|o| o.id.to_string()) .or_else(|| { node.func .as_attribute_expr() - .and_then(|attr| attr.attr.parse().ok()) + .map(|attr| attr.attr.to_string()) }) .ok_or(())?; // `obj_name` is the object (receiver) - i.e., `obj` in `obj.method()` - let obj_name: String = if let Expr::Attribute(rustpython_ast::ExprAttribute { - value, .. - }) = node.func.as_ref() - { - if let Expr::Name(expr_name) = value.as_ref() { - expr_name.id.parse().map_err(|_| ())? - } else { - return Err(()); - } - } else { - return Err(()); - }; + let obj_name: String = node + .func + .as_attribute_expr() + .and_then(|attr| attr.value.as_name_expr()) + .map(|name| name.id.to_string()) + .ok_or(())?; if obj_name == "wmill" { // Continue } else if let Some((kind, ref path, ref schema)) = self.var_identifiers.get(&obj_name) { if ident == "query" { - let expr_name = node.args.get(0).or_else(|| { - node.keywords + let expr_name = node.arguments.args.get(0).or_else(|| { + node.arguments + .keywords .iter() .find(|kw| kw.arg.as_deref() == Some("name")) .map(|kw| &kw.value) }); let sql = match expr_name { - Some(Expr::Constant(ExprConstant { value: Constant::Str(sql), .. })) => sql, + Some(Expr::StringLiteral(sql)) => sql.value.to_str(), _ => return Err(()), }; // We use the SQL parser to detect RW, specific tables, etc. @@ -224,7 +236,7 @@ impl AssetsFinder { *kind, path, schema.as_deref(), - &sql, + sql, ); match sql_assets { Ok(Some(sql_assets)) => self.assets.extend(sql_assets), @@ -252,9 +264,10 @@ impl AssetsFinder { }; let arg_val = match arg { - Arg(i, name) => node.args.get(i).or_else(|| { + Arg(i, name) => node.arguments.args.get(i).or_else(|| { // Get arg by name - node.keywords + node.arguments + .keywords .iter() .find(|kw| kw.arg.as_deref() == Some(name)) .map(|kw| &kw.value) @@ -270,8 +283,9 @@ impl AssetsFinder { self.assets .push(ParseAssetsResult { kind, path, access_type, columns: None }); } - Some(Expr::Constant(ExprConstant { value: Constant::Str(value), .. })) => { - let path = parse_asset_syntax(&value, false) + Some(Expr::StringLiteral(value)) => { + let value = value.value.to_str(); + let path = parse_asset_syntax(value, false) .map(|(_, p)| p) .unwrap_or(std::borrow::Cow::Borrowed(value)); self.assets.push(ParseAssetsResult { @@ -291,26 +305,24 @@ impl AssetsFinder { struct Arg(usize, &'static str); /// Extract a string-literal keyword argument, e.g. `s3="value"` in a call. -fn keyword_str_value(keywords: &[rustpython_ast::Keyword], name: &str) -> Option { +fn keyword_str_value(keywords: &[Keyword], name: &str) -> Option { keywords .iter() .find(|kw| kw.arg.as_deref() == Some(name)) .and_then(|kw| match &kw.value { - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => Some(s.clone()), + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), _ => None, }) } /// Extract a string-literal value from a dict literal, e.g. `{"s3": "value"}`. -fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option { - dict.keys +fn dict_str_value(dict: &ExprDict, name: &str) -> Option { + dict.items .iter() - .zip(dict.values.iter()) - .find_map(|(key, value)| match (key.as_ref()?, value) { - ( - Expr::Constant(ExprConstant { value: Constant::Str(k), .. }), - Expr::Constant(ExprConstant { value: Constant::Str(v), .. }), - ) if k.as_str() == name => Some(v.clone()), + .find_map(|item| match (item.key.as_ref()?, &item.value) { + (Expr::StringLiteral(k), Expr::StringLiteral(v)) if k.value.to_str() == name => { + Some(v.value.to_str().to_string()) + } _ => None, }) } @@ -328,12 +340,13 @@ fn dict_str_value(dict: &rustpython_ast::ExprDict, name: &str) -> Option /// matches the TS object form and the `# on s3:///…` trigger form exactly. fn s3_object_arg_path(expr: &Expr) -> Option { let uri = match expr { - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => { + Expr::StringLiteral(s) => { + let s = s.value.to_str(); match s .strip_prefix("s3://") .and_then(|rest| rest.split_once('/')) { - Some((_, key)) if !key.is_empty() => s.clone(), + Some((_, key)) if !key.is_empty() => s.to_string(), _ => return None, } } @@ -347,8 +360,9 @@ fn s3_object_arg_path(expr: &Expr) -> Option { if func_name != "S3Object" { return None; } - let key = keyword_str_value(&call.keywords, "s3")?; - let storage = keyword_str_value(&call.keywords, "storage").unwrap_or_default(); + let key = keyword_str_value(&call.arguments.keywords, "s3")?; + let storage = + keyword_str_value(&call.arguments.keywords, "storage").unwrap_or_default(); format!("s3://{storage}/{key}") } Expr::Dict(dict) => { @@ -822,4 +836,62 @@ def main(): },]) ); } + + #[test] + fn test_py_asset_parser_recent_syntax() { + let input = r#" +import wmill +lazy import polars + +type Rows = list[dict] + +def main[T](name: str): + try: + with wmill.load_s3_file_reader('s3:///in.csv') as f: + f.read() + except ValueError, TypeError: + pass + greeting = t"hello {name}" + return f"{wmill.get_resource("u/user/pg")["host"]}" +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()), + Ok(vec![ + ParseAssetsResult { + kind: AssetKind::S3Object, + path: "/in.csv".to_string(), + access_type: Some(R), + columns: None, + }, + ParseAssetsResult { + kind: AssetKind::Resource, + path: "u/user/pg".to_string(), + access_type: None, + columns: None, + }, + ]) + ); + } + + #[test] + fn test_py_asset_parser_schema_qualified_db_without_table() { + let input = r#" +import wmill +def main(): + db = wmill.datatable('dt:sch') + db.query("ATTACH 'ducklake://other' AS o") + db.query("SELECT * FROM t") +"#; + let s = parse_assets(input).map(|o| o.assets); + assert_eq!( + s.map_err(|e| e.to_string()) + .map(|a| a.into_iter().map(|a| (a.kind, a.path)).collect::>()), + Ok(vec![ + (AssetKind::DataTable, "dt".to_string()), + (AssetKind::DataTable, "dt/sch.t".to_string()), + (AssetKind::Ducklake, "other".to_string()), + ]) + ); + } } diff --git a/backend/parsers/windmill-parser-py-imports/Cargo.toml b/backend/parsers/windmill-parser-py-imports/Cargo.toml index 565e1422bb..2ef83f5ac9 100644 --- a/backend/parsers/windmill-parser-py-imports/Cargo.toml +++ b/backend/parsers/windmill-parser-py-imports/Cargo.toml @@ -23,7 +23,8 @@ tracing.workspace = true [dependencies] windmill-parser.workspace = true -rustpython-parser.workspace = true +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true phf.workspace = true itertools.workspace = true serde_json.workspace = true diff --git a/backend/parsers/windmill-parser-py-imports/src/lib.rs b/backend/parsers/windmill-parser-py-imports/src/lib.rs index de7f51bc79..5c8de9790f 100644 --- a/backend/parsers/windmill-parser-py-imports/src/lib.rs +++ b/backend/parsers/windmill-parser-py-imports/src/lib.rs @@ -13,9 +13,9 @@ use async_recursion::async_recursion; use itertools::Itertools; use lazy_static::lazy_static; #[cfg(not(target_arch = "wasm32"))] -use std::str::FromStr; -#[cfg(not(target_arch = "wasm32"))] use std::collections::HashMap; +#[cfg(not(target_arch = "wasm32"))] +use std::str::FromStr; use mapping::{FULL_IMPORTS_MAP, SHORT_IMPORTS_MAP}; #[cfg(not(target_arch = "wasm32"))] @@ -23,11 +23,8 @@ use regex::Regex; #[cfg(target_arch = "wasm32")] use regex_lite::Regex; -use rustpython_parser::{ - ast::{Stmt, StmtImport, StmtImportFrom, Suite}, - text_size::TextRange, - Parse, -}; +use ruff_python_ast::{Stmt, StmtImport, StmtImportFrom}; +use ruff_python_parser::parse_module; #[cfg(not(target_arch = "wasm32"))] use sqlx::{Pool, Postgres}; #[cfg(not(target_arch = "wasm32"))] @@ -160,6 +157,7 @@ pub struct ImportPin { } pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result> { + let full_code = code; // Use regex to safely find the main function definition let mut code = DEF_MAIN_RE .split(code) @@ -186,16 +184,22 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result (code, parsed.into_suite()), + // The split lands on any line reading like the definition of `main`, one inside a + // string included: what precedes it is then not a module, while the whole file is. + Err(e) => match parse_module(full_code) { + Ok(parsed) => (full_code.to_string(), parsed.into_suite()), + Err(_) => anyhow::bail!("Error parsing code for imports: {}", e.to_string()), + }, + }; // Note: We're still using the original code for finding pins, // as the TextRange values from the parsed AST would be based on code_with_fake_main // but we want to match against the original code - let find_pin = |range: TextRange, key: String| { + let find_pin = |stmt_end: usize, key: String| { let hs = code .chars() - .skip(range.end().to_usize()) + .skip(stmt_end) .take_while(|e| *e != '\n') .collect::(); @@ -226,9 +230,9 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result = ast .into_iter() .filter_map(|x| match x { - Stmt::Import(StmtImport { names, range }) => names + Stmt::Import(StmtImport { names, range, .. }) => names .get(0) - .and_then(|al| find_pin(range, al.name.to_string())) + .and_then(|al| find_pin(range.end().to_usize(), al.name.to_string())) .or(Some( names .into_iter() @@ -239,15 +243,11 @@ pub fn parse_code_for_imports(code: &str, path: &str) -> anyhow::Result>(), )), - Stmt::ImportFrom(StmtImportFrom { level: Some(i), module, .. }) if i.to_u32() > 0 => { - Some(process_import( - module.map(|x| x.to_string()), - path, - i.to_usize(), - )) - } + Stmt::ImportFrom(StmtImportFrom { level, module, .. }) if level > 0 => Some( + process_import(module.map(|x| x.to_string()), path, level as usize), + ), Stmt::ImportFrom(StmtImportFrom { level: _, module, range, .. }) => find_pin( - range, + range.end().to_usize(), module.clone().map(|x| x.to_string()).unwrap_or_default(), ) .or(Some(process_import(module.map(|x| x.to_string()), path, 0))), @@ -351,13 +351,16 @@ async fn parse_python_imports_inner( temp_script_refs: &Option>, ) -> error::Result> { tracing::debug!("Parsing python imports for path: {}", path); - let PythonAnnotations { py310, py311, py312, py313, .. } = PythonAnnotations::parse(&code); + let PythonAnnotations { py310, py311, py312, py313, py314, py315, .. } = + PythonAnnotations::parse(&code); tracing::debug!( - "Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}", + "Found python annotations - py310: {}, py311: {}, py312: {}, py313: {}, py314: {}, py315: {}", py310, py311, py312, - py313 + py313, + py314, + py315 ); let mut push_version_specifiers = |perform, unparsed: String| -> error::Result<()> { @@ -373,6 +376,8 @@ async fn parse_python_imports_inner( push_version_specifiers(py311, "==3.11.*".to_owned())?; push_version_specifiers(py312, "==3.12.*".to_owned())?; push_version_specifiers(py313, "==3.13.*".to_owned())?; + push_version_specifiers(py314, "==3.14.*".to_owned())?; + push_version_specifiers(py315, "==3.15.*".to_owned())?; for x in code.lines() { if x.starts_with("# py:") || x.starts_with("#py:") { @@ -513,8 +518,14 @@ async fn parse_python_imports_inner( let mut nested = match n { NImport::Relative(rpath) => { // First try to get content from temp_script_refs cache if available - let code_from_cache = if let Some(hash) = temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) { - tracing::debug!("Found relative import '{}' in temp_script_refs with hash '{}'", rpath, hash); + let code_from_cache = if let Some(hash) = + temp_script_refs.as_ref().and_then(|dt| dt.get(&rpath)) + { + tracing::debug!( + "Found relative import '{}' in temp_script_refs with hash '{}'", + rpath, + hash + ); match windmill_common::cache::raw_script_temp::load(hash.clone(), db).await { Ok(content) => Some(content), Err(e) => { @@ -529,19 +540,17 @@ async fn parse_python_imports_inner( // Use cached content if available, otherwise fall back to deployed script let code = match code_from_cache { Some(content) => content, - None => { - sqlx::query_scalar!( - r#" + None => sqlx::query_scalar!( + r#" SELECT content FROM script WHERE path = $1 AND workspace_id = $2 AND archived = false ORDER BY created_at DESC LIMIT 1 "#, - &rpath, - w_id - ) - .fetch_optional(db) - .await? - .unwrap_or_else(|| "".to_string()) - } + &rpath, + w_id + ) + .fetch_optional(db) + .await? + .unwrap_or_else(|| "".to_string()), }; if already_visited.contains(&rpath) { diff --git a/backend/parsers/windmill-parser-py-imports/src/mapping.rs b/backend/parsers/windmill-parser-py-imports/src/mapping.rs index 6246a1f156..90f3fab99c 100644 --- a/backend/parsers/windmill-parser-py-imports/src/mapping.rs +++ b/backend/parsers/windmill-parser-py-imports/src/mapping.rs @@ -383,5 +383,10 @@ pub static SHORT_IMPORTS_MAP: PyMap = phf_map! { "vt" => "vt-py", "grpc" => "grpcio", "serpapi" => "google-search-results", + "pptx" => "python-pptx", + "OpenSSL" => "pyOpenSSL", + "win32api" => "pywin32", + "ruamel" => "ruamel.yaml", + "speech-recognition" => "SpeechRecognition", // Add new entry here ^ }; diff --git a/backend/parsers/windmill-parser-py-imports/tests/tests.rs b/backend/parsers/windmill-parser-py-imports/tests/tests.rs index c853c8e5b4..9086b29d47 100644 --- a/backend/parsers/windmill-parser-py-imports/tests/tests.rs +++ b/backend/parsers/windmill-parser-py-imports/tests/tests.rs @@ -1,7 +1,9 @@ mod tests { use sqlx::{Pool, Postgres}; - use windmill_parser_py_imports::parse_python_imports; + use windmill_parser_py_imports::{ + parse_code_for_imports, parse_python_imports, ImportPin, NImport, + }; #[sqlx::test(fixtures("base"))] async fn test_parse_python_imports(db: Pool) -> anyhow::Result<()> { @@ -117,4 +119,87 @@ def main(): Ok(()) } + + fn auto(pkg: &str) -> NImport { + NImport::Auto { pkg: pkg.to_string(), key: None } + } + + #[test] + fn test_parse_lazy_imports() -> anyhow::Result<()> { + let code = " +lazy import requests +lazy import numpy # pin: numpy==2.1.0 +lazy from pandas import DataFrame +lazy from f.shared import helper +lazy = 1 + +def main(): + pass +"; + assert_eq!( + parse_code_for_imports(code, "f/foo/bar")?, + vec![ + NImport::Pin { + pins: vec![ImportPin { + pkg: "numpy==2.1.0".to_string(), + path: "f/foo/bar".to_string() + }], + key: "numpy".to_string() + }, + auto("pandas"), + auto("requests"), + NImport::Relative("f/shared".to_string()), + ] + ); + Ok(()) + } + + #[test] + fn test_parse_imports_recent_syntax_before_main() -> anyhow::Result<()> { + let code = r#" +import requests + +type Alias = list[int] +NAME = "x" +D = {"a": 1} +L = [[1], [2]] +TEMPLATE = t"hello {NAME}" +PEP701 = f"{D["a"]} {'\n'.join(["x"])}" +PEP798 = [*a for a in L] + +try: + import ujson +except ImportError, ValueError: + pass + +def main(): + pass +"#; + assert_eq!( + parse_code_for_imports(code, "f/foo/bar")?, + vec![auto("requests")] + ); + Ok(()) + } + + #[test] + fn test_parse_imports_main_lookalike_in_docstring() -> anyhow::Result<()> { + for example in ["def main[T](x: T):", "def main(x):"] { + let code = format!( + "\"\"\"Usage example:\n{example}\n return x\n\"\"\"\nimport requests # pin: requests==2.32.0\n\ndef main(value: str):\n return value\n" + ); + assert_eq!( + parse_code_for_imports(&code, "f/foo/bar")?, + vec![NImport::Pin { + pins: vec![ImportPin { + pkg: "requests==2.32.0".to_string(), + path: "f/foo/bar".to_string() + }], + key: "requests".to_string() + }], + "{code}" + ); + } + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-py/Cargo.toml b/backend/parsers/windmill-parser-py/Cargo.toml index 0c50e0f8d8..6971fe8fcd 100644 --- a/backend/parsers/windmill-parser-py/Cargo.toml +++ b/backend/parsers/windmill-parser-py/Cargo.toml @@ -10,8 +10,8 @@ path = "./src/lib.rs" [dependencies] windmill-parser.workspace = true -rustpython-parser.workspace = true +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true itertools.workspace = true serde_json.workspace = true anyhow.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 086a354d02..5a9eabc33e 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -13,13 +13,11 @@ use itertools::Itertools; use serde_json::json; use windmill_parser::{json_to_typ, Arg, MainArgSignature, ObjectType, Typ}; -use rustpython_parser::{ - ast::{ - Constant, Expr, ExprAttribute, ExprConstant, ExprDict, ExprList, ExprName, Stmt, - StmtAssign, StmtAsyncFunctionDef, StmtClassDef, StmtFunctionDef, Suite, - }, - Parse, +use ruff_python_ast::{ + Expr, ExprAttribute, ExprDict, ExprList, ExprName, Number, Parameters, Stmt, StmtAssign, + StmtClassDef, StmtFunctionDef, }; +use ruff_python_parser::parse_module; pub mod pydantic_parser; @@ -81,22 +79,24 @@ fn should_parse_for_models(code: &str) -> bool { || code.contains("import dataclasses") } +/// What follows `def ` or `async def ` on a line opening that function's +/// parameters or type parameters. +fn strip_def<'a>(line: &'a str, name: &str) -> Option<&'a str> { + let rest = line + .strip_prefix("async def ") + .or_else(|| line.strip_prefix("def "))? + .strip_prefix(name)?; + (rest.starts_with('(') || rest.starts_with('[')).then_some(rest) +} + fn filter_non_main(code: &str, main_name: &str) -> String { - let def_main = format!("def {}(", main_name); - let async_def_main = format!("async def {}(", main_name); let mut filtered_code = String::new(); let mut code_iter = code.split("\n"); let mut remaining: String = String::new(); while let Some(line) = code_iter.next() { - if line.starts_with(&async_def_main) { - filtered_code += &async_def_main; - remaining += line.strip_prefix(&async_def_main).unwrap(); - remaining += "\n"; - remaining += &code_iter.join("\n"); - break; - } else if line.starts_with(&def_main) { - filtered_code += &def_main; - remaining += line.strip_prefix(&def_main).unwrap(); + if let Some(rest) = strip_def(line, main_name) { + filtered_code += &line[..line.len() - rest.len()]; + remaining += rest; remaining += "\n"; remaining += &code_iter.join("\n"); break; @@ -106,6 +106,26 @@ fn filter_non_main(code: &str, main_name: &str) -> String { return String::new(); } let mut chars = remaining.chars(); + + // PEP 695 type parameters, `def main[T](x: T)`, sit between the name and the parameters. + if remaining.starts_with('[') { + let mut open_brackets = 0; + for c in chars.by_ref() { + if c == '[' { + open_brackets += 1; + } else if c == ']' { + open_brackets -= 1; + } + filtered_code.push(c); + if open_brackets == 0 { + break; + } + } + } + if chars.next() != Some('(') { + return String::new(); + } + filtered_code.push('('); let mut open_parens = 1; while let Some(c) = chars.next() { @@ -183,15 +203,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { let filtered_code = filter_relevant_statements(code); - let ast = match Suite::parse(&filtered_code, "main.py") { - Ok(ast) => ast, + let ast = match parse_module(&filtered_code) { + Ok(parsed) => parsed.into_suite(), Err(_) => return CodeMetadata { enums, descriptions }, }; for stmt in ast { match stmt { - Stmt::ClassDef(StmtClassDef { name, body, bases, .. }) if has_enum => { - let is_enum = bases.iter().any(|base| { + Stmt::ClassDef(class_def) if has_enum => { + let is_enum = class_def.bases().iter().any(|base| { matches!(base, Expr::Name(ExprName { id, .. }) if id == "Enum" || id == "IntEnum" || id == "StrEnum" || id == "Flag" || id == "IntFlag") @@ -201,17 +221,15 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { let mut values = Vec::new(); let mut members = HashMap::new(); + let StmtClassDef { name, body, .. } = class_def; for item in body { if let Stmt::Assign(StmtAssign { targets, value, .. }) = item { if let Some(Expr::Name(ExprName { id: target_name, .. })) = targets.first() { if !target_name.starts_with('_') { - if let Expr::Constant(ExprConstant { - value: Constant::Str(val), - .. - }) = value.as_ref() - { + if let Expr::StringLiteral(val) = value.as_ref() { + let val = val.value.to_str(); values.push(val.to_string()); members.insert(target_name.to_string(), val.to_string()); } @@ -225,15 +243,13 @@ fn extract_code_metadata(code: &str, main_name: &str) -> CodeMetadata { } } } - Stmt::FunctionDef(StmtFunctionDef { name: func_name, body, .. }) if has_docstring => { - if &func_name == main_name { + Stmt::FunctionDef(StmtFunctionDef { + name: func_name, body, is_async: false, .. + }) if has_docstring => { + if func_name.as_str() == main_name { if let Some(Stmt::Expr(expr_stmt)) = body.first() { - if let Expr::Constant(ExprConstant { - value: Constant::Str(docstring), - .. - }) = expr_stmt.value.as_ref() - { - descriptions = parse_docstring_args(docstring); + if let Expr::StringLiteral(docstring) = expr_stmt.value.as_ref() { + descriptions = parse_docstring_args(docstring.value.to_str()); } } } @@ -294,6 +310,27 @@ fn parse_docstring_args(docstring: &str) -> HashMap { descriptions } +/// A line that reads as the definition of `main_name` can also sit inside a string, a docstring +/// showing an example for instance. When several lines do, only the parsed module tells which +/// one is the function. +fn main_params_among_lookalikes(code: &str, main_name: &str) -> Option { + code.split('\n') + .filter(|line| strip_def(line, main_name).is_some()) + .nth(1)?; + parse_module(code) + .ok()? + .into_suite() + .into_iter() + .find_map(|x| match x { + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => + { + Some(*parameters) + } + _ => None, + }) +} + /// skip_params is a micro optimization for when we just want to find the main /// function without parsing all the params. pub fn parse_python_signature( @@ -310,41 +347,39 @@ pub fn parse_python_signature( // - If no models: parse only the filtered main function let (params, module) = if should_parse_for_models(code) { // Parse full code once for both Pydantic detection and signature extraction - let ast = Suite::parse(code, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(code) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); // Extract main function from full AST let params = ast.iter().find_map(|x| match x { - Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if name == &main_name => { - Some(args.as_ref().clone()) - } - Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. }) - if name == &main_name => + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => { - Some(args.as_ref().clone()) + Some(parameters.as_ref().clone()) } _ => None, }); // Keep AST for Pydantic/dataclass detection (params, Some(ast)) + } else if let Some(params) = main_params_among_lookalikes(code, &main_name) { + (Some(params), None) } else { // No models detected - parse only the filtered main function let filtered_code = filter_non_main(code, &main_name); if filtered_code.is_empty() { (None, None) } else { - let ast = Suite::parse(&filtered_code, "main.py") - .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; + let ast = parse_module(&filtered_code) + .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))? + .into_suite(); let params = ast.into_iter().find_map(|x| match x { - Stmt::FunctionDef(StmtFunctionDef { name, args, .. }) if &name == &main_name => { - Some(*args) - } - Stmt::AsyncFunctionDef(StmtAsyncFunctionDef { name, args, .. }) - if &name == &main_name => + Stmt::FunctionDef(StmtFunctionDef { name, parameters, .. }) + if name.as_str() == main_name => { - Some(*args) + Some(*parameters) } _ => None, }); @@ -378,8 +413,6 @@ pub fn parse_python_signature( if !skip_params && params.is_some() { let params = params.unwrap(); - let def_arg_start = params.args.len() - params.defaults().count(); - // Two-pass approach for lazy metadata extraction: // Pass 1: Parse types without enum info to determine if metadata is needed // Pass 2: Re-parse unknown types with metadata only if necessary @@ -392,11 +425,11 @@ pub fn parse_python_signature( .iter() .enumerate() .map(|(i, x)| { - let arg_name = x.as_arg().arg.to_string(); + let arg_name = x.parameter.name.to_string(); let (typ, has_default) = x - .as_arg() + .parameter .annotation - .as_ref() + .as_deref() .map_or((Typ::Unknown, false), |e| { parse_expr(e, &empty_enums, module_ref) }); @@ -423,21 +456,16 @@ pub fn parse_python_signature( .into_iter() .map(|(i, arg_name, mut typ, mut has_default)| { if matches!(typ, Typ::Resource(_)) && !metadata.enums.is_empty() { - if let Some(annotation) = params.args[i].as_arg().annotation.as_ref() { + if let Some(annotation) = params.args[i].parameter.annotation.as_deref() { (typ, has_default) = parse_expr(annotation, &metadata.enums, module_ref); } } - let default = if i >= def_arg_start { - params - .defaults() - .nth(i - def_arg_start) - .map(|expr| to_value(expr, &metadata.enums)) - .flatten() - } else { - None - }; + let default = params.args[i] + .default + .as_deref() + .and_then(|expr| to_value(expr, &metadata.enums)); let should_get_type_from_default = match &typ { Typ::Unknown => true, @@ -465,10 +493,10 @@ pub fn parse_python_signature( // debounce accumulation support. Falls back to docstring // description if no union array pattern is found. let union_otyp = params.args[i] - .as_arg() + .parameter .annotation - .as_ref() - .and_then(|ann| detect_py_union_array_otyp(ann.as_ref())); + .as_deref() + .and_then(detect_py_union_array_otyp); Arg { otyp: union_otyp.or_else(|| { @@ -509,12 +537,8 @@ pub fn parse_python_signature( } } -fn parse_expr( - e: &Box, - enums: &HashMap, - module: Option<&[Stmt]>, -) -> (Typ, bool) { - match e.as_ref() { +fn parse_expr(e: &Expr, enums: &HashMap, module: Option<&[Stmt]>) -> (Typ, bool) { + match e { Expr::Name(ExprName { id, .. }) => (parse_typ(id.as_ref(), enums, module), false), Expr::Attribute(x) => { if let Some(name) = x.value.as_name_expr() { @@ -531,12 +555,9 @@ fn parse_expr( } } Expr::BinOp(x) => { - if matches!( - x.right.as_ref(), - Expr::Constant(ExprConstant { value: Constant::None, .. }) - ) { + if x.right.is_none_literal_expr() { (parse_expr(&x.left, enums, module).0, true) - } else if detect_py_union_array_otyp(e.as_ref()).is_some() { + } else if detect_py_union_array_otyp(e).is_some() { // T | list[T] — parsed type is Unknown; otyp is set separately (Typ::Unknown, false) } else { @@ -552,7 +573,7 @@ fn parse_expr( .elts .iter() .map(|x| match x { - Expr::Constant(c) => c.value.as_str().map(|x| x.to_string()), + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), _ => None, }) .filter_map(|x| x) @@ -625,15 +646,26 @@ fn map_resource_name(x: &str) -> String { } } -fn to_value(et: &Expr, enums: &HashMap) -> Option { +fn to_value(et: &Expr, enums: &HashMap) -> Option { match et { - Expr::Constant(ExprConstant { value, .. }) => Some(constant_to_value(value)), - Expr::Dict(ExprDict { keys, values, .. }) => { - let v = keys - .into_iter() - .zip(values) - .map(|(k, v)| { - let key = k + Expr::NoneLiteral(_) => Some(json!(null)), + Expr::BooleanLiteral(b) => Some(json!(b.value)), + Expr::StringLiteral(s) => Some(json!(s.value.to_str())), + Expr::BytesLiteral(b) => Some(json!(b.value.bytes().collect::>())), + Expr::NumberLiteral(n) => Some(match &n.value { + Number::Int(i) => { + serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")) + } + Number::Float(f) => json!(f), + Number::Complex { real, imag } => json!([real, imag]), + }), + Expr::EllipsisLiteral(_) => Some(json!("...")), + Expr::Dict(ExprDict { items, .. }) => { + let v = items + .iter() + .map(|item| { + let key = item + .key .as_ref() .map(|e| to_value(e, enums)) .flatten() @@ -642,7 +674,7 @@ fn to_value(et: &Expr, enums: &HashMap) -> Option None, }) .unwrap_or_else(|| "no_key".to_string()); - (key, to_value(&v, enums)) + (key, to_value(&item.value, enums)) }) .collect::>(); Some(json!(v)) @@ -672,20 +704,6 @@ fn to_value(et: &Expr, enums: &HashMap) -> Option serde_json::Value { - match c { - Constant::None => json!(null), - Constant::Bool(b) => json!(b), - Constant::Str(s) => json!(s), - Constant::Bytes(b) => json!(b), - Constant::Int(i) => serde_json::from_str(&i.to_string()).unwrap_or(json!("invalid number")), - Constant::Tuple(t) => json!(t.iter().map(constant_to_value).collect::>()), - Constant::Float(f) => json!(f), - Constant::Complex { real, imag } => json!([real, imag]), - Constant::Ellipsis => json!("..."), - } -} - #[cfg(test)] mod tests { @@ -1323,4 +1341,89 @@ def main(color: Color = Color.RED): ); Ok(()) } + + #[test] + fn test_parse_python_sig_type_params() -> anyhow::Result<()> { + for code in [ + "def main[T](x: str, y: int = 3):\n return x\n", + "async def main[T: (int, str), *Ts](x: str, y: int = 3):\n return x\n", + // parsed whole rather than pre-filtered down to `main` + "from dataclasses import dataclass\n\ndef main[T](x: str, y: int = 3):\n return x\n", + ] { + let result = parse_python_signature(code, None, false)?; + assert_eq!(result.auto_kind, None, "{code}"); + assert_eq!( + result + .args + .iter() + .map(|a| (a.name.as_str(), &a.typ, &a.default)) + .collect::>(), + vec![ + ("x", &Typ::Str(None), &None), + ("y", &Typ::Int, &Some(json!(3))) + ], + "{code}" + ); + } + Ok(()) + } + + // A script with a model class is parsed whole, so syntax anywhere in it reaches the parser. + #[test] + fn test_parse_python_sig_recent_syntax() -> anyhow::Result<()> { + let code = r#" +from dataclasses import dataclass +lazy import requests +lazy from pandas import DataFrame + +type Alias = list[int] + +@dataclass +class Point: + x: int + +try: + import a +except ImportError, ValueError: + pass + +D = {"a": 1} +L = [[1], [2]] +PEP701 = f"{D["a"]} {'\n'.join(["x"])}" +PEP798 = [*a for a in L] + +def main(p: Point, name: str = "world"): + return t"hello {name}" +"#; + let result = parse_python_signature(code, None, false)?; + assert_eq!(result.args.len(), 2); + assert_eq!( + result.args[0].typ, + Typ::Object(ObjectType::new( + Some("Point".to_string()), + Some(vec![windmill_parser::ObjectProperty { + key: "x".to_string(), + typ: Box::new(Typ::Int) + }]) + )) + ); + assert_eq!(result.args[1].default, Some(json!("world"))); + Ok(()) + } + + #[test] + fn test_parse_python_sig_main_lookalike_in_docstring() -> anyhow::Result<()> { + for example in ["def main[T](x: T):", "def main(x: int, y: int):"] { + let code = format!( + "\"\"\"Usage example:\n{example}\n return x\n\"\"\"\n\ndef main(value: str):\n return value\n" + ); + let result = parse_python_signature(&code, None, false)?; + assert_eq!( + result.args.iter().map(|a| &a.name).collect::>(), + vec!["value"], + "{code}" + ); + } + Ok(()) + } } diff --git a/backend/parsers/windmill-parser-py/src/pydantic_parser.rs b/backend/parsers/windmill-parser-py/src/pydantic_parser.rs index e229d97a8a..78a0a3c354 100644 --- a/backend/parsers/windmill-parser-py/src/pydantic_parser.rs +++ b/backend/parsers/windmill-parser-py/src/pydantic_parser.rs @@ -11,8 +11,8 @@ //! This module provides functionality to detect and parse Pydantic models and Python //! dataclasses from Python AST, enabling automatic UI generation for complex data structures. -use rustpython_parser::ast::{ - Constant, Expr, ExprAttribute, ExprCall, ExprConstant, ExprName, ExprTuple, Stmt, StmtAnnAssign, +use ruff_python_ast::{ + Decorator, Expr, ExprAttribute, ExprCall, ExprName, ExprTuple, Stmt, StmtAnnAssign, }; use std::collections::HashSet; use windmill_parser::{ObjectProperty, ObjectType, Typ}; @@ -64,7 +64,7 @@ fn detect_model_type_impl( // Mark as being visited visited.insert(class_name.to_string()); - let result = if is_pydantic_base(&class_def.bases) { + let result = if is_pydantic_base(class_def.bases()) { // Pydantic BaseModel parse_model_fields(&class_def.body, class_def.name.as_str(), module, visited) } else if has_dataclass_decorator(&class_def.decorator_list) @@ -115,9 +115,9 @@ fn is_pydantic_base(bases: &[Expr]) -> bool { } /// Checks if a class has @dataclass decorator (standard library) -fn has_dataclass_decorator(decorators: &[Expr]) -> bool { +fn has_dataclass_decorator(decorators: &[Decorator]) -> bool { for decorator in decorators { - match decorator { + match &decorator.expression { // Match: @dataclass Expr::Name(ExprName { id, .. }) if id.as_str() == "dataclass" => { return true; @@ -160,9 +160,9 @@ fn has_dataclass_decorator(decorators: &[Expr]) -> bool { } /// Checks if a class has @pydantic.dataclasses.dataclass decorator (Pydantic v2) -fn has_pydantic_dataclass_decorator(decorators: &[Expr]) -> bool { +fn has_pydantic_dataclass_decorator(decorators: &[Decorator]) -> bool { for decorator in decorators { - match decorator { + match &decorator.expression { // Match: @pydantic.dataclasses.dataclass Expr::Attribute(ExprAttribute { attr, value, .. }) if attr.as_str() == "dataclass" => { if let Expr::Attribute(ExprAttribute { @@ -350,10 +350,10 @@ fn extract_field_type( // String annotations: "ForwardRef" (forward references) // Not fully supported - return Unknown with warning - Expr::Constant(ExprConstant { value: Constant::Str(s), .. }) => { + Expr::StringLiteral(s) => { eprintln!( "Forward references like \"{forward_ref}\" are not yet supported, treating as Unknown", - forward_ref = s, + forward_ref = s.value.to_str(), ); Typ::Unknown } diff --git a/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs b/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs index b29c6c99ac..4d77da15bb 100644 --- a/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs +++ b/backend/parsers/windmill-parser-sql-asset/src/asset_parser_utils.rs @@ -20,13 +20,16 @@ pub fn parse_wmill_sdk_sql_assets( Ok(mut sql_assets) => { if let Some(schema) = schema { for asset in &mut sql_assets.assets { - if asset.kind == kind && asset.path.starts_with(asset_name) { - asset.path = format!( - "{}/{}.{}", - asset_name, - schema, - &asset.path[asset_name.len() + 1..] - ); + if asset.kind != kind { + continue; + } + // A statement can name the database alone, with no table to qualify. + let table = asset + .path + .strip_prefix(asset_name) + .and_then(|rest| rest.strip_prefix('/')); + if let Some(table) = table { + asset.path = format!("{asset_name}/{schema}.{table}"); } } } diff --git a/backend/parsers/windmill-parser-wac/Cargo.toml b/backend/parsers/windmill-parser-wac/Cargo.toml index d354b1c653..21ee286db9 100644 --- a/backend/parsers/windmill-parser-wac/Cargo.toml +++ b/backend/parsers/windmill-parser-wac/Cargo.toml @@ -9,8 +9,9 @@ name = "windmill_parser_wac" path = "./src/lib.rs" [dependencies] -rustpython-parser.workspace = true -rustpython-ast = { version = "0.4.0", features = ["visitor"] } +ruff_python_parser.workspace = true +ruff_python_ast.workspace = true +ruff_text_size.workspace = true swc_common.workspace = true swc_ecma_parser.workspace = true swc_ecma_ast.workspace = true diff --git a/backend/parsers/windmill-parser-wac/src/python.rs b/backend/parsers/windmill-parser-wac/src/python.rs index f91067b711..e8fb72ff2f 100644 --- a/backend/parsers/windmill-parser-wac/src/python.rs +++ b/backend/parsers/windmill-parser-wac/src/python.rs @@ -1,12 +1,12 @@ use std::collections::HashMap; -use rustpython_parser::{ - ast::{ - Expr, ExprAwait, ExprCall, ExprName, Stmt, StmtExpr, StmtFor, StmtIf, StmtReturn, StmtTry, - StmtTryStar, StmtWhile, - }, - Parse, +use ruff_python_ast::{ + CmpOp, ElifElseClause, ExceptHandler, Expr, ExprAttribute, ExprAwait, ExprCall, ExprName, + Number, Parameters, Stmt, StmtExpr, StmtFor, StmtFunctionDef, StmtIf, StmtReturn, StmtTry, + StmtWhile, }; +use ruff_python_parser::parse_module; +use ruff_text_size::{Ranged, TextSize}; use crate::dag::{DagEdge, DagNode, DagNodeType, Param, WorkflowDag}; use crate::validation::{self, CompileError}; @@ -42,9 +42,9 @@ type TaskFunctions = HashMap>; fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { let mut tasks = HashMap::new(); for stmt in stmts { - if let Stmt::AsyncFunctionDef(func) = stmt { + if let Stmt::FunctionDef(func @ StmtFunctionDef { is_async: true, .. }) = stmt { for dec in &func.decorator_list { - match dec { + match &dec.expression { // @task (bare decorator) Expr::Name(ExprName { id, .. }) if id.as_str() == "task" => { tasks.insert(func.name.to_string(), None); @@ -68,13 +68,9 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { if let Expr::Name(ExprName { id, .. }) = call.func.as_ref() { if id.as_str() == "task_script" || id.as_str() == "task_flow" { // Extract the path from the first positional argument - let path = call.args.first().and_then(|arg| { - if let Expr::Constant(c) = arg { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } - } - None + let path = call.arguments.args.first().and_then(|arg| match arg { + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), + _ => None, }); // Extract variable name from target if let Some(Expr::Name(ExprName { id: var_name, .. })) = @@ -92,13 +88,11 @@ fn collect_task_functions(stmts: &[Stmt]) -> TaskFunctions { /// Extract the `path=` keyword argument from a `@task(path="...")` call. fn extract_task_path_kwarg(call: &ExprCall) -> Option { - for kw in &call.keywords { + for kw in &call.arguments.keywords { if let Some(ref arg) = kw.arg { if arg.as_str() == "path" { - if let Expr::Constant(c) = &kw.value { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } + if let Expr::StringLiteral(s) = &kw.value { + return Some(s.value.to_str().to_string()); } } } @@ -150,10 +144,9 @@ impl WacWalker { fn line_of_expr(&self, expr: &Expr) -> usize { let offset = match expr { - Expr::Call(c) => c.range.start().to_usize(), - Expr::Await(a) => a.range.start().to_usize(), - Expr::Attribute(a) => a.range.start().to_usize(), - Expr::Name(n) => n.range.start().to_usize(), + Expr::Call(_) | Expr::Await(_) | Expr::Attribute(_) | Expr::Name(_) => { + expr.start().to_usize() + } _ => 0, }; self.line_index.line_of(offset) @@ -161,17 +154,15 @@ impl WacWalker { fn line_of_stmt(&self, stmt: &Stmt) -> usize { let offset = match stmt { - Stmt::If(s) => s.range.start().to_usize(), - Stmt::For(s) => s.range.start().to_usize(), - Stmt::While(s) => s.range.start().to_usize(), - Stmt::Return(s) => s.range.start().to_usize(), - Stmt::Expr(s) => s.range.start().to_usize(), - Stmt::Try(s) => s.range.start().to_usize(), - Stmt::TryStar(s) => s.range.start().to_usize(), - Stmt::Assign(s) => s.range.start().to_usize(), - Stmt::AnnAssign(s) => s.range.start().to_usize(), - Stmt::FunctionDef(s) => s.range.start().to_usize(), - Stmt::AsyncFunctionDef(s) => s.range.start().to_usize(), + Stmt::If(_) + | Stmt::For(_) + | Stmt::While(_) + | Stmt::Return(_) + | Stmt::Expr(_) + | Stmt::Try(_) + | Stmt::Assign(_) + | Stmt::AnnAssign(_) + | Stmt::FunctionDef(_) => stmt.start().to_usize(), _ => 0, }; self.line_index.line_of(offset) @@ -190,9 +181,7 @@ impl WacWalker { /// Check if an expression is `asyncio.gather(...)` call fn is_asyncio_gather_call(expr: &Expr) -> bool { if let Expr::Call(call) = expr { - if let Expr::Attribute(rustpython_parser::ast::ExprAttribute { value, attr, .. }) = - call.func.as_ref() - { + if let Expr::Attribute(ExprAttribute { value, attr, .. }) = call.func.as_ref() { if attr.as_str() == "gather" { if let Expr::Name(ExprName { id, .. }) = value.as_ref() { return id.as_str() == "asyncio"; @@ -222,20 +211,20 @@ impl WacWalker { fn expr_to_source(expr: &Expr) -> String { match expr { Expr::Compare(c) => { - let left = Self::expr_to_source(&c.left); - if let Some(comparator) = c.comparators.first() { + let left = Self::expr_to_source(&c.operands[0]); + if let Some(comparator) = c.comparators().first() { let right = Self::expr_to_source(comparator); let op = match c.ops.first() { - Some(rustpython_parser::ast::CmpOp::Gt) => ">", - Some(rustpython_parser::ast::CmpOp::Lt) => "<", - Some(rustpython_parser::ast::CmpOp::GtE) => ">=", - Some(rustpython_parser::ast::CmpOp::LtE) => "<=", - Some(rustpython_parser::ast::CmpOp::Eq) => "==", - Some(rustpython_parser::ast::CmpOp::NotEq) => "!=", - Some(rustpython_parser::ast::CmpOp::In) => "in", - Some(rustpython_parser::ast::CmpOp::NotIn) => "not in", - Some(rustpython_parser::ast::CmpOp::Is) => "is", - Some(rustpython_parser::ast::CmpOp::IsNot) => "is not", + Some(CmpOp::Gt) => ">", + Some(CmpOp::Lt) => "<", + Some(CmpOp::GtE) => ">=", + Some(CmpOp::LtE) => "<=", + Some(CmpOp::Eq) => "==", + Some(CmpOp::NotEq) => "!=", + Some(CmpOp::In) => "in", + Some(CmpOp::NotIn) => "not in", + Some(CmpOp::Is) => "is", + Some(CmpOp::IsNot) => "is not", None => "?", }; format!("{left} {op} {right}") @@ -253,14 +242,14 @@ impl WacWalker { format!("{value}.{}", a.attr) } Expr::Name(n) => n.id.to_string(), - Expr::Constant(c) => match &c.value { - rustpython_parser::ast::Constant::Str(s) => format!("\"{s}\""), - rustpython_parser::ast::Constant::Int(i) => i.to_string(), - rustpython_parser::ast::Constant::Float(f) => f.to_string(), - rustpython_parser::ast::Constant::Bool(b) => b.to_string(), - rustpython_parser::ast::Constant::None => "None".to_string(), - _ => "...".to_string(), + Expr::StringLiteral(s) => format!("\"{}\"", s.value.to_str()), + Expr::NumberLiteral(n) => match &n.value { + Number::Int(i) => i.to_string(), + Number::Float(f) => f.to_string(), + Number::Complex { .. } => "...".to_string(), }, + Expr::BooleanLiteral(b) => b.value.to_string(), + Expr::NoneLiteral(_) => "None".to_string(), _ => "...".to_string(), } } @@ -279,8 +268,16 @@ impl WacWalker { match stmt { Stmt::Expr(StmtExpr { value, .. }) => self.expr_contains_step(value), Stmt::Assign(a) => self.expr_contains_step(&a.value), - Stmt::If(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), - Stmt::For(s) => self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse), + Stmt::If(s) => { + self.body_contains_step(&s.body) + || s.elif_else_clauses + .iter() + .any(|clause| self.body_contains_step(&clause.body)) + } + // `async for` is not modelled: its body is skipped like any unsupported statement + Stmt::For(s) if !s.is_async => { + self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse) + } Stmt::While(s) => { self.body_contains_step(&s.body) || self.body_contains_step(&s.orelse) } @@ -289,19 +286,7 @@ impl WacWalker { || self.body_contains_step(&s.orelse) || self.body_contains_step(&s.finalbody) || s.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } - }) - } - Stmt::TryStar(s) => { - self.body_contains_step(&s.body) - || self.body_contains_step(&s.orelse) - || self.body_contains_step(&s.finalbody) - || s.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } + ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body), }) } Stmt::Return(_) => false, @@ -319,11 +304,12 @@ impl WacWalker { match expr { Expr::Await(ExprAwait { value, .. }) => self.expr_contains_step(value), Expr::Call(call) => { - if self.is_task_fn_call(&Expr::Call(call.clone())) { - return true; - } - if Self::is_asyncio_gather_call(&Expr::Call(call.clone())) { - return call.args.iter().any(|a| self.expr_contains_step(a)); + if Self::is_asyncio_gather_call(expr) { + return call + .arguments + .args + .iter() + .any(|a| self.expr_contains_step(a)); } false } @@ -370,12 +356,11 @@ impl WacWalker { Stmt::Expr(StmtExpr { value, .. }) => self.walk_expr_stmt(value), Stmt::Assign(a) => self.walk_expr_stmt(&a.value), Stmt::If(if_stmt) => self.walk_if(if_stmt), - Stmt::For(for_stmt) => self.walk_for(for_stmt), + Stmt::For(for_stmt) if !for_stmt.is_async => self.walk_for(for_stmt), Stmt::While(while_stmt) => self.walk_while(while_stmt), Stmt::Try(try_stmt) => self.walk_try(try_stmt), - Stmt::TryStar(try_stmt) => self.walk_try_star(try_stmt), Stmt::Return(ret) => self.walk_return(ret), - Stmt::FunctionDef(_) | Stmt::AsyncFunctionDef(_) => { + Stmt::FunctionDef(_) => { if self.stmt_contains_step(stmt) { self.errors.push(validation::error_step_in_nested_function( self.line_of_stmt(stmt), @@ -392,7 +377,7 @@ impl WacWalker { if let Expr::Await(ExprAwait { value, .. }) = expr { // await task_fn(...) if let Expr::Call(call) = value.as_ref() { - if self.is_task_fn_call(&Expr::Call(call.clone())) { + if self.is_task_fn_call(value) { return self.emit_step(call, expr); } // Check for SDK-level calls: step(), sleep(), wait_for_approval() @@ -430,15 +415,12 @@ impl WacWalker { "step" => { // step("name", fn) — extract the name from the first string argument let name = call + .arguments .args .first() - .and_then(|arg| { - if let Expr::Constant(c) = arg { - if let rustpython_parser::ast::Constant::Str(s) = &c.value { - return Some(s.to_string()); - } - } - None + .and_then(|arg| match arg { + Expr::StringLiteral(s) => Some(s.value.to_str().to_string()), + _ => None, }) .unwrap_or_else(|| "step".to_string()); let id = self.next_id(); @@ -452,6 +434,7 @@ impl WacWalker { } "sleep" => { let seconds = call + .arguments .args .first() .map(|arg| Self::expr_to_source(arg)) @@ -517,10 +500,10 @@ impl WacWalker { }); let mut step_ids = Vec::new(); - for arg in &gather_call.args { + for arg in &gather_call.arguments.args { // Each arg should be task_fn(...) if let Expr::Call(call) = arg { - if self.is_task_fn_call(&Expr::Call(call.clone())) { + if self.is_task_fn_call(arg) { let (name, script) = self .extract_step_info_from_task_call(call) .unwrap_or(("unknown".into(), "unknown".into())); @@ -553,15 +536,33 @@ impl WacWalker { } fn walk_if(&mut self, if_stmt: &StmtIf) -> Option<(String, String)> { - let has_steps_in_body = self.body_contains_step(&if_stmt.body); - let has_steps_in_else = self.body_contains_step(&if_stmt.orelse); + self.walk_if_chain( + if_stmt.start(), + &if_stmt.test, + &if_stmt.body, + &if_stmt.elif_else_clauses, + ) + } + + /// An `elif` is laid out as an `if` nested alone in the `else` of the clause before it. + fn walk_if_chain( + &mut self, + start: TextSize, + test: &Expr, + body: &[Stmt], + clauses: &[ElifElseClause], + ) -> Option<(String, String)> { + let has_steps_in_body = self.body_contains_step(body); + let has_steps_in_else = clauses + .iter() + .any(|clause| self.body_contains_step(&clause.body)); if !has_steps_in_body && !has_steps_in_else { return None; } - let line = self.line_index.line_of(if_stmt.range.start().to_usize()); - let condition_source = Self::expr_to_source(&if_stmt.test); + let line = self.line_index.line_of(start.to_usize()); + let condition_source = Self::expr_to_source(test); let branch_id = self.next_id(); let branch_node_id = self.add_node(DagNode { @@ -573,15 +574,19 @@ impl WacWalker { let mut last_ids = Vec::new(); - if let Some((true_first, true_last)) = self.walk_body(&if_stmt.body) { + if let Some((true_first, true_last)) = self.walk_body(body) { self.add_edge(&branch_node_id, &true_first, Some("true".to_string())); last_ids.push(true_last); } else { last_ids.push(branch_node_id.clone()); } - if !if_stmt.orelse.is_empty() { - if let Some((else_first, else_last)) = self.walk_body(&if_stmt.orelse) { + if let Some((clause, clauses)) = clauses.split_first() { + let orelse = match &clause.test { + Some(test) => self.walk_if_chain(clause.start(), test, &clause.body, clauses), + None => self.walk_body(&clause.body), + }; + if let Some((else_first, else_last)) = orelse { self.add_edge(&branch_node_id, &else_first, Some("false".to_string())); last_ids.push(else_last); } else { @@ -611,7 +616,7 @@ impl WacWalker { return None; } - let line = self.line_index.line_of(for_stmt.range.start().to_usize()); + let line = self.line_index.line_of(for_stmt.start().to_usize()); let iter_source = Self::expr_to_source(&for_stmt.iter); let start_id = self.next_id(); @@ -644,7 +649,7 @@ impl WacWalker { return None; } - let line = self.line_index.line_of(while_stmt.range.start().to_usize()); + let line = self.line_index.line_of(while_stmt.start().to_usize()); let condition = Self::expr_to_source(&while_stmt.test); let start_id = self.next_id(); @@ -677,39 +682,14 @@ impl WacWalker { || self.body_contains_step(&try_stmt.orelse) || self.body_contains_step(&try_stmt.finalbody) || try_stmt.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } + ExceptHandler::ExceptHandler(eh) => self.body_contains_step(&eh.body), }); if !has_steps { return None; } - let line = self.line_index.line_of(try_stmt.range.start().to_usize()); - self.emit_try_catch_branch( - &try_stmt.body, - &try_stmt.handlers, - &try_stmt.finalbody, - line, - ) - } - - fn walk_try_star(&mut self, try_stmt: &StmtTryStar) -> Option<(String, String)> { - let has_steps = self.body_contains_step(&try_stmt.body) - || self.body_contains_step(&try_stmt.orelse) - || self.body_contains_step(&try_stmt.finalbody) - || try_stmt.handlers.iter().any(|h| match h { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { - self.body_contains_step(&eh.body) - } - }); - - if !has_steps { - return None; - } - - let line = self.line_index.line_of(try_stmt.range.start().to_usize()); + let line = self.line_index.line_of(try_stmt.start().to_usize()); self.emit_try_catch_branch( &try_stmt.body, &try_stmt.handlers, @@ -721,7 +701,7 @@ impl WacWalker { fn emit_try_catch_branch( &mut self, try_body: &[Stmt], - handlers: &[rustpython_parser::ast::ExceptHandler], + handlers: &[ExceptHandler], finally_body: &[Stmt], line: usize, ) -> Option<(String, String)> { @@ -746,7 +726,7 @@ impl WacWalker { // Except handlers for handler in handlers { match handler { - rustpython_parser::ast::ExceptHandler::ExceptHandler(eh) => { + ExceptHandler::ExceptHandler(eh) => { if let Some((catch_first, catch_last)) = self.walk_body(&eh.body) { self.add_edge(&branch_node_id, &catch_first, Some("except".to_string())); last_ids.push(catch_last); @@ -783,7 +763,7 @@ impl WacWalker { } fn walk_return(&mut self, ret: &StmtReturn) -> Option<(String, String)> { - let line = self.line_index.line_of(ret.range.start().to_usize()); + let line = self.line_index.line_of(ret.start().to_usize()); let id = self.next_id(); let node_id = self.add_node(DagNode { id: id.clone(), @@ -796,12 +776,12 @@ impl WacWalker { } /// Extract workflow function parameters (no longer skips ctx) -fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec { +fn extract_params(args: &Parameters) -> Vec { let mut params = Vec::new(); for arg_with_default in args.args.iter().chain(args.posonlyargs.iter()) { - let name = arg_with_default.def.arg.to_string(); + let name = arg_with_default.parameter.name.to_string(); let typ = arg_with_default - .def + .parameter .annotation .as_ref() .map(|ann| WacWalker::expr_to_source(ann)); @@ -811,59 +791,34 @@ fn extract_params(args: &rustpython_parser::ast::Arguments) -> Vec { } pub fn parse_python_workflow(code: &str) -> Result> { - let ast = rustpython_parser::ast::Suite::parse(code, "") - .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])?; + let ast = parse_module(code) + .map_err(|e| vec![CompileError { message: format!("Parse error: {e}"), line: 0 }])? + .into_suite(); // First pass: collect @task functions let task_functions = collect_task_functions(&ast); - // Find the @workflow async def - let workflow_fn = ast.iter().find_map(|stmt| { - if let Stmt::AsyncFunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - return Some(func); + let workflow_fns = || { + ast.iter().filter_map(|stmt| match stmt { + Stmt::FunctionDef(func) + if func.decorator_list.iter().any(|dec| { + matches!(&dec.expression, Expr::Name(ExprName { id, .. }) if id.as_str() == "workflow") + }) => + { + Some(func) } - } - // Also check non-async for error reporting - if let Stmt::FunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - return None; // Will be reported as not-async below - } - } - None - }); + _ => None, + }) + }; - // Check for non-async workflow function - let non_async_workflow = ast.iter().find_map(|stmt| { - if let Stmt::FunctionDef(func) = stmt { - let has_workflow_decorator = func.decorator_list.iter().any(|dec| { - if let Expr::Name(ExprName { id, .. }) = dec { - id.as_str() == "workflow" - } else { - false - } - }); - if has_workflow_decorator { - let line_index = LineIndex::new(code); - return Some(line_index.line_of(func.range.start().to_usize())); - } - } - None - }); + // Find the @workflow async def + let workflow_fn = workflow_fns().find(|func| func.is_async); + + // Check for non-async workflow function. The statement starts at its first decorator, + // the name sits on the `def` line the error points at. + let non_async_workflow = workflow_fns() + .find(|func| !func.is_async) + .map(|func| LineIndex::new(code).line_of(func.name.start().to_usize())); if let Some(line) = non_async_workflow { if workflow_fn.is_none() { @@ -875,7 +830,7 @@ pub fn parse_python_workflow(code: &str) -> Result String { hasher.update(code.as_bytes()); format!("{:x}", hasher.finalize()) } - -trait ToUsize { - fn to_usize(self) -> usize; -} - -impl ToUsize for rustpython_parser::text_size::TextSize { - fn to_usize(self) -> usize { - u32::from(self) as usize - } -} diff --git a/backend/parsers/windmill-parser-wac/tests/python_tests.rs b/backend/parsers/windmill-parser-wac/tests/python_tests.rs index cf117da9e7..c0de489153 100644 --- a/backend/parsers/windmill-parser-wac/tests/python_tests.rs +++ b/backend/parsers/windmill-parser-wac/tests/python_tests.rs @@ -363,3 +363,77 @@ async def main(x: str): )); assert!(matches!(dag.nodes[5].node_type, DagNodeType::Return)); } + +#[test] +fn test_elif_chain_nests_branches() { + let code = r#" +from wmill import workflow, task + +@task +async def a(): ... +@task +async def b(): ... +@task +async def c(): ... + +@workflow +async def main(x: int): + if x > 2: + await a() + elif x == 2: + await b() + else: + await c() +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + let branches: Vec<(&str, usize)> = dag + .nodes + .iter() + .filter_map(|n| match &n.node_type { + DagNodeType::Branch { condition_source } => Some((condition_source.as_str(), n.line)), + _ => None, + }) + .collect(); + assert_eq!(branches, vec![("x > 2", 13), ("x == 2", 15)]); + + // The `elif` branch hangs off the `false` edge of the `if` before it + let (outer, inner) = (&dag.nodes[0].id, &dag.nodes[2].id); + assert!(dag + .edges + .iter() + .any(|e| &e.from == outer && &e.to == inner && e.label.as_deref() == Some("false"))); + assert_eq!( + dag.nodes + .iter() + .filter(|n| matches!(n.node_type, DagNodeType::Merge)) + .count(), + 2 + ); +} + +#[test] +fn test_workflow_with_recent_syntax() { + let code = r#" +from wmill import workflow, task +lazy import polars + +type Rows = list[dict] + +@task +async def load[T](rows: T): ... + +@workflow +async def main(name: str): + try: + await load(t"hello {name}") + except ValueError, TypeError: + pass +"#; + + let dag = parse_python_workflow(code).expect("should parse"); + assert!(dag + .nodes + .iter() + .any(|n| matches!(&n.node_type, DagNodeType::Step { name, .. } if name == "load"))); +} diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index 1bdbdd6097..75c10528df 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -254,6 +254,35 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "attribute-derive" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f9ef7a68684657ce9da341c934c73b63df7a3585a61e366d84c84ed5d3d167" +dependencies = [ + "attribute-derive-macro", + "manyhow", + "proc-macro2", + "quote", + "syn 3.0.7", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300d426e5e73549125bb8e4649bdd5508f82c94ddf898fb0517246e71dfb8e98" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn 3.0.7", +] + [[package]] name = "autocfg" version = "1.5.0" @@ -428,7 +457,7 @@ dependencies = [ "proc-macro2", "quote", "regex", - "rustc-hash 2.1.1", + "rustc-hash", "shlex", "syn 2.0.117", ] @@ -620,6 +649,15 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659" +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.56" @@ -670,6 +708,18 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "char_str" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9cba78a5abb515610e10af297615434b019755b5d7be14e559ff5340dff50c4" +dependencies = [ + "castaway", + "get-size2", + "itoa", + "ryu", +] + [[package]] name = "chrono" version = "0.4.44" @@ -779,6 +829,12 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" +[[package]] +name = "collection_literals" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + [[package]] name = "colorchoice" version = "1.0.4" @@ -795,6 +851,19 @@ dependencies = [ "memchr", ] +[[package]] +name = "compact_str" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79fcda08c33bb58b97008b2cdada6622500e949e060f5913361763121abd2416" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "static_assertions", + "zmij", +] + [[package]] name = "compression-codecs" version = "0.4.37" @@ -1019,12 +1088,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - [[package]] name = "crypto-common" version = "0.1.7" @@ -1093,27 +1156,6 @@ dependencies = [ "powerfmt", ] -[[package]] -name = "derive_more" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a9b99b9cbbe49445b21764dc0625032a89b145a2642e67603e1c936f5458d05" -dependencies = [ - "derive_more-impl", -] - -[[package]] -name = "derive_more-impl" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7330aeadfbe296029522e6c40f315320aba36fc43a5b3632f3795348f3bd22" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", - "unicode-xid", -] - [[package]] name = "des" version = "0.7.0" @@ -1196,6 +1238,12 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1aaf95b3e5c8f23aa320147307562d361db0ae0d51242340f558153b4eb2439b" +[[package]] +name = "drop_bomb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9bda8e21c04aca2ae33ffc2fd8c23134f3cac46db123ba97bd9d3f3b8a4a85e1" + [[package]] name = "dunce" version = "1.0.5" @@ -1502,6 +1550,31 @@ dependencies = [ "version_check", ] +[[package]] +name = "get-size-derive2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28f0daa179069133df60523e0f59d0a6bd2d3db9ccae01b630896d269c4a802e" +dependencies = [ + "attribute-derive", + "quote", + "syn 3.0.7", +] + +[[package]] +name = "get-size2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "050a5d07359ebfac7f0b74916d94cfa30575e45fdc7e9bfbb0f76d5d4203325a" +dependencies = [ + "compact_str", + "get-size-derive2", + "hashbrown 0.17.1", + "ordermap", + "smallvec", + "thin-vec", +] + [[package]] name = "gethostname" version = "0.4.3" @@ -1686,6 +1759,12 @@ version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + [[package]] name = "hashlink" version = "0.10.0" @@ -1758,7 +1837,7 @@ dependencies = [ "hashbrown 0.14.5", "new_debug_unreachable", "once_cell", - "rustc-hash 2.1.1", + "rustc-hash", "triomphe", ] @@ -1870,7 +1949,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2", "system-configuration", "tokio", "tower-service", @@ -2012,16 +2091,22 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.1", "serde", "serde_core", ] +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + [[package]] name = "inventory" version = "0.3.22" @@ -2082,15 +2167,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" -[[package]] -name = "itertools" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" -dependencies = [ - "either", -] - [[package]] name = "itertools" version = "0.13.0" @@ -2109,6 +2185,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.17" @@ -2212,12 +2297,6 @@ version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37" -[[package]] -name = "lalrpop-util" -version = "0.20.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507460a910eb7b32ee961886ff48539633b788a36b65692b95f225b844c82553" - [[package]] name = "lazy_static" version = "1.5.0" @@ -2388,61 +2467,26 @@ dependencies = [ ] [[package]] -name = "malachite" -version = "0.4.22" +name = "manyhow" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fbdf9cb251732db30a7200ebb6ae5d22fe8e11397364416617d2c2cf0c51cb5" +checksum = "8c69ea6e469ca785c846b25dd69eac9e9c916b51d8ada1f212866787ee1da12d" dependencies = [ - "malachite-base", - "malachite-nz", - "malachite-q", + "manyhow-macros", + "proc-macro2", + "quote", + "syn 3.0.7", ] [[package]] -name = "malachite-base" -version = "0.4.22" +name = "manyhow-macros" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ea0ed76adf7defc1a92240b5c36d5368cfe9251640dcce5bd2d0b7c1fd87aeb" +checksum = "22fc5c1b2c4000056dae3619fea36e79ce79dabfee4907d2dcad6612b189e370" dependencies = [ - "hashbrown 0.14.5", - "itertools 0.11.0", - "libm", - "ryu", -] - -[[package]] -name = "malachite-bigint" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d149aaa2965d70381709d9df4c7ee1fc0de1c614a4efc2ee356f5e43d68749f8" -dependencies = [ - "derive_more", - "malachite", - "num-integer", - "num-traits", - "paste", -] - -[[package]] -name = "malachite-nz" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34a79feebb2bc9aa7762047c8e5495269a367da6b5a90a99882a0aeeac1841f7" -dependencies = [ - "itertools 0.11.0", - "libm", - "malachite-base", -] - -[[package]] -name = "malachite-q" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50f235d5747b1256b47620f5640c2a17a88c7569eebdf27cd9cb130e1a619191" -dependencies = [ - "itertools 0.11.0", - "malachite-base", - "malachite-nz", + "proc-macro-utils", + "proc-macro2", + "quote", ] [[package]] @@ -2949,6 +2993,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordermap" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16a756ddcdeb6c0efc8927bdf0ed112885815a35ba350472a467be5104f01ca1" +dependencies = [ + "indexmap", +] + [[package]] name = "owo-colors" version = "4.3.0" @@ -2984,12 +3037,6 @@ dependencies = [ "windows-link", ] -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - [[package]] name = "pem" version = "3.0.6" @@ -3286,6 +3333,17 @@ dependencies = [ "version_check", ] +[[package]] +name = "proc-macro-utils" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" +dependencies = [ + "proc-macro2", + "quote", + "smallvec", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -3401,9 +3459,9 @@ dependencies = [ "pin-project-lite", "quinn-proto", "quinn-udp", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", - "socket2 0.6.3", + "socket2", "thiserror 2.0.18", "tokio", "tracing", @@ -3422,7 +3480,7 @@ dependencies = [ "lru-slab", "rand 0.9.5", "ring", - "rustc-hash 2.1.1", + "rustc-hash", "rustls", "rustls-pki-types", "slab", @@ -3441,7 +3499,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2", "tracing", "windows-sys 0.60.2", ] @@ -3455,6 +3513,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "quote-use" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48356e493823dc3c1d7e9fc7b3ff204ecec48afbdd718fc4134091b88eb63f3d" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cf1c5de6a7cc231c71e03c77aa4c7019097859ea7029156b54d2894a8430704" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", + "syn 3.0.7", +] + [[package]] name = "r-efi" version = "5.3.0" @@ -3835,6 +3915,106 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ruff_python_ast" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15434b06df7d930fad2818639a4e0fec4af9c770121c45a96b72da7feaae9ead" +dependencies = [ + "aho-corasick", + "arrayvec", + "bitflags", + "char_str", + "compact_str", + "get-size2", + "is-macro", + "memchr", + "ruff_python_trivia", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "thin-vec", + "thiserror 2.0.18", +] + +[[package]] +name = "ruff_python_parser" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37a22811d752aa9e255d7dd72524d14352cf9bc654ef79b399bf1eeeb6640fbb" +dependencies = [ + "bitflags", + "bstr", + "drop_bomb", + "get-size2", + "hashbrown 0.17.1", + "memchr", + "ruff_python_ast", + "ruff_python_trivia", + "ruff_text_size", + "ruff_unicode_names2", + "rustc-hash", + "stacker", + "static_assertions", + "thin-vec", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_python_trivia" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0322a5059fe0feb2e010ca6f31c24ac289f6b7e9e38359539491b10c57f1c5bc" +dependencies = [ + "itertools 0.15.0", + "memchr", + "ruff_source_file", + "ruff_text_size", + "rustc-hash", + "unicode-ident", + "unicode-normalization", +] + +[[package]] +name = "ruff_source_file" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6ae809a289b5c791f82aa45df33f2a92323e7a1c1d265c10e9ac884027a2e43" +dependencies = [ + "memchr", + "ruff_text_size", +] + +[[package]] +name = "ruff_text_size" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf6344d6915ced9316312c1ee3f24c4300c54295bc4c3b8f4e95b633ae5443b" +dependencies = [ + "get-size2", +] + +[[package]] +name = "ruff_unicode_names2" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13418b8fd029d597e35dedc36e91f47122d26075e4c86557574d9441753fb375" +dependencies = [ + "phf 0.11.3", + "ruff_unicode_names2_generator", +] + +[[package]] +name = "ruff_unicode_names2_generator" +version = "0.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37fd82af49d5685d9054e68185987acaaf62542f671b6df8935155664876bc39" +dependencies = [ + "phf_codegen", + "rand 0.8.6", +] + [[package]] name = "rust_decimal" version = "1.40.0" @@ -3857,12 +4037,6 @@ version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b50b8869d9fc858ce7266cce0194bd74df58b9d0e3f6df3a9fc8eb470d95c09d" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustc-hash" version = "2.1.1" @@ -3971,63 +4145,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "rustpython-ast" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cdaf8ee5c1473b993b398c174641d3aa9da847af36e8d5eb8291930b72f31a5" -dependencies = [ - "is-macro", - "malachite-bigint", - "rustpython-parser-core", - "static_assertions", -] - -[[package]] -name = "rustpython-parser" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868f724daac0caf9bd36d38caf45819905193a901e8f1c983345a68e18fb2abb" -dependencies = [ - "anyhow", - "is-macro", - "itertools 0.11.0", - "lalrpop-util", - "log", - "malachite-bigint", - "num-traits", - "phf 0.11.3", - "phf_codegen", - "rustc-hash 1.1.0", - "rustpython-ast", - "rustpython-parser-core", - "tiny-keccak", - "unic-emoji-char", - "unic-ucd-ident", - "unicode_names2", -] - -[[package]] -name = "rustpython-parser-core" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4b6c12fa273825edc7bccd9a734f0ad5ba4b8a2f4da5ff7efe946f066d0f4ad" -dependencies = [ - "is-macro", - "memchr", - "rustpython-parser-vendored", -] - -[[package]] -name = "rustpython-parser-vendored" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fcea49a4630a3a5d940f4d514dc4f575ed63c14c3e3ed07146634aed7f67a6" -dependencies = [ - "memchr", - "once_cell", -] - [[package]] name = "rustversion" version = "1.0.22" @@ -4040,6 +4157,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "ryu-js" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714" + [[package]] name = "same-file" version = "1.0.6" @@ -4409,16 +4532,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "socket2" -version = "0.5.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "socket2" version = "0.6.3" @@ -4680,15 +4793,15 @@ checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "stacker" -version = "0.1.23" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d74a23609d509411d10e2176dc2a4346e3b4aea2e7b1869f19fdedbc71c013" +checksum = "707f49d46706bacf8a2b00d51dace3f9de527c13eec3778f570c411f89e69967" dependencies = [ "cc", "cfg-if", "libc", "psm", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4830,7 +4943,7 @@ dependencies = [ "new_debug_unreachable", "num-bigint", "once_cell", - "rustc-hash 2.1.1", + "rustc-hash", "serde", "siphasher 0.3.11", "swc_atoms", @@ -4852,7 +4965,7 @@ dependencies = [ "num-bigint", "once_cell", "phf 0.11.3", - "rustc-hash 2.1.1", + "rustc-hash", "string_enum", "swc_atoms", "swc_common", @@ -4871,7 +4984,7 @@ dependencies = [ "either", "num-bigint", "phf 0.11.3", - "rustc-hash 2.1.1", + "rustc-hash", "seq-macro", "serde", "smallvec", @@ -4974,6 +5087,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d62a2e0561533f2ca2561d0cf27fd9fedb640a1bf2616ff5d5c80d99017faadc" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -5102,6 +5226,12 @@ dependencies = [ "unicode-width 0.2.2", ] +[[package]] +name = "thin-vec" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6a4b9ba8738cb4a4f399d37e266becfd475e75eb73425b87a05a2f2039ba63e" + [[package]] name = "thiserror" version = "1.0.69" @@ -5193,15 +5323,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tiny-keccak" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" -dependencies = [ - "crunchy", -] - [[package]] name = "tinystr" version = "0.8.2" @@ -5229,9 +5350,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.46.1" +version = "1.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cc3a2344dafbe23a245241fe8b09735b521110d30fcefbbd5feb1797ca35d17" +checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038" dependencies = [ "backtrace", "bytes", @@ -5242,10 +5363,10 @@ dependencies = [ "pin-project-lite", "signal-hook-registry", "slab", - "socket2 0.5.10", + "socket2", "tokio-macros", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5289,7 +5410,7 @@ dependencies = [ "postgres-protocol", "postgres-types", "rand 0.10.1", - "socket2 0.6.3", + "socket2", "tokio", "tokio-util", "whoami 2.1.0", @@ -5325,6 +5446,7 @@ dependencies = [ "bytes", "futures-core", "futures-sink", + "futures-util", "pin-project-lite", "tokio", ] @@ -5663,17 +5785,6 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" -[[package]] -name = "unic-emoji-char" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b07221e68897210270a38bde4babb655869637af0f69407f96053a34f76494d" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-category" version = "0.9.0" @@ -5686,17 +5797,6 @@ dependencies = [ "unic-ucd-version", ] -[[package]] -name = "unic-ucd-ident" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e230a37c0381caa9219d67cf063aa3a375ffed5bf541a452db16e744bdab6987" -dependencies = [ - "unic-char-property", - "unic-char-range", - "unic-ucd-version", -] - [[package]] name = "unic-ucd-version" version = "0.9.0" @@ -5775,28 +5875,6 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" -[[package]] -name = "unicode_names2" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" -dependencies = [ - "phf 0.11.3", - "unicode_names2_generator", -] - -[[package]] -name = "unicode_names2_generator" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91e5b84611016120197efd7dc93ef76774f4e084cd73c9fb3ea4a86c570c56e" -dependencies = [ - "getopts", - "log", - "phf_codegen", - "rand 0.8.6", -] - [[package]] name = "unscanny" version = "0.1.0" @@ -6216,7 +6294,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.828.0" +version = "1.830.0" dependencies = [ "aho-corasick", "anyhow", @@ -6260,12 +6338,14 @@ dependencies = [ "pin-project-lite", "pkcs1", "postgres-native-tls", + "postgres-protocol", "quick_cache", "rand 0.9.5", "regex", "reqwest", "reqwest-middleware", "reqwest-retry", + "ryu-js", "semver", "serde", "serde_json", @@ -6302,7 +6382,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.828.0" +version = "1.830.0" dependencies = [ "proc-macro2", "quote", @@ -6314,7 +6394,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.828.0" +version = "1.830.0" dependencies = [ "convert_case", "serde", @@ -6323,7 +6403,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6335,7 +6415,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -6347,7 +6427,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "gosyn", @@ -6359,7 +6439,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6371,7 +6451,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -6383,7 +6463,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "nu-parser", @@ -6394,7 +6474,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6405,30 +6485,30 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "itertools 0.14.0", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde_json", "windmill-parser", ] [[package]] name = "windmill-parser-py-asset" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "windmill-parser", "windmill-parser-sql-asset", ] [[package]] name = "windmill-parser-py-imports" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "async-recursion", @@ -6438,7 +6518,8 @@ dependencies = [ "phf 0.11.3", "regex", "regex-lite", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", "serde", "serde_json", "sqlx", @@ -6450,7 +6531,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "serde_json", @@ -6462,7 +6543,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6557,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "convert_case", @@ -6493,7 +6574,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6587,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "serde", @@ -6518,7 +6599,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6536,7 +6617,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6552,11 +6633,12 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", - "rustpython-ast", - "rustpython-parser", + "ruff_python_ast", + "ruff_python_parser", + "ruff_text_size", "serde", "serde_json", "sha2 0.10.9", @@ -6568,7 +6650,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6600,7 +6682,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "lazy_static", @@ -6614,7 +6696,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.828.0" +version = "1.830.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index b2fd850217..8aebf20597 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.828.0" +version = "1.830.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/main.rs b/backend/src/main.rs index 81104d6764..b8fdd9353f 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -63,12 +63,14 @@ use windmill_common::{ SANDBOX_IMAGE_CACHE_MAX_MB_SETTING, SANDBOX_IMAGE_DEFAULT_REGISTRY_SETTING, SANDBOX_IMAGE_MAX_SIZE_MB_SETTING, SANDBOX_IMAGE_PULL_POLICY_SETTING, SANDBOX_REGISTRY_AUTH_SETTING, SCIM_TOKEN_SETTING, SERVICE_LOG_RETENTION_SECS_SETTING, - SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, - UV_EXCLUDE_NEWER_SETTING, UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, + SMTP_SETTING, STORE_AUDIT_LOGS_S3_SETTING, SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING, + TEAMS_SETTING, TIMEOUT_WAIT_RESULT_SETTING, UV_EXCLUDE_NEWER_SETTING, + UV_INDEX_STRATEGY_SETTING, UV_PYTHON_INSTALL_MIRROR_SETTING, WORKSPACE_FAIRNESS_DURATION_SECS_SETTING, WORKSPACE_FAIRNESS_ENABLED_SETTING, WORKSPACE_FAIRNESS_MAX_PERCENT_SETTING, WORKSPACE_FAIRNESS_MIN_TOTAL_SETTING, WORKSPACE_MAX_QUEUED_JOBS_SETTING, WORKSPACE_REGISTRIES_SETTING, }, + hub_resource_types, scripts::ScriptLang, stats_oss::schedule_stats, triggers::TriggerKind, @@ -121,7 +123,8 @@ use windmill_worker::{ BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, CSHARP_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, JAVA_CACHE_DIR, NU_CACHE_DIR, POWERSHELL_CACHE_DIR, PY310_CACHE_DIR, PY311_CACHE_DIR, PY312_CACHE_DIR, PY313_CACHE_DIR, - RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR, TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, + PY314_CACHE_DIR, PY315_CACHE_DIR, RUBY_CACHE_DIR, RUST_CACHE_DIR, R_CACHE_DIR, + TAR_JAVA_CACHE_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -401,111 +404,16 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { Ok(()) } -/// Raw resource type from hub API (schema is a JSON string) -#[derive(serde::Deserialize)] -struct HubResourceTypeRaw { - pub id: i64, - pub name: String, - pub schema: Option, - pub app: String, - pub description: Option, - /// Absent from hubs predating the column, and from caches written before it. - #[serde(default)] - pub format_extension: Option, - /// Doubly optional, so a hub predating the field (no key) is told apart from a type the - /// hub leaves unnamed (null). - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - pub display_name: Option>, -} - -/// Processed resource type with parsed schema -#[derive(serde::Deserialize, serde::Serialize, Clone)] -pub struct HubResourceType { - pub id: i64, - pub name: String, - pub schema: Option, - pub app: String, - pub description: Option, - /// Doubly optional on purpose. A cache written before this column has no key at - /// all (`None`) and must leave the stored extension alone; one written since - /// always writes the key, so an explicit null (`Some(None)`) is the hub genuinely - /// dropping it and must clear. A single `Option` conflates the two, and picking - /// either meaning breaks the other — as does plain serde, which folds `null` - /// into the outer `None`, hence the wrapping deserializer. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option", - skip_serializing_if = "Option::is_none" - )] - pub format_extension: Option>, - /// Doubly optional like `format_extension`: a cache written before the field leaves the - /// stored name alone, while a null from the hub clears it. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option", - skip_serializing_if = "Option::is_none" - )] - pub display_name: Option>, -} - -const HUB_RT_CACHE_FILE: &str = "resource_types.json"; - async fn cache_hub_resource_types() -> anyhow::Result<()> { println!("Caching resource types from hub..."); - let response = HTTP_CLIENT - .get(format!("{}/resource_types/list", DEFAULT_HUB_BASE_URL)) - .header("Accept", "application/json") - .send() - .await - .with_context(|| "Failed to fetch resource types from hub")?; - - if !response.status().is_success() { - anyhow::bail!( - "Failed to fetch resource types from hub: {}", - response.status() - ); - } - - let raw_types: Vec = response - .json::>() - .await - .with_context(|| "Failed to parse resource types from hub")?; - - // Parse schema strings into JSON values - let resource_types: Vec = raw_types - .into_iter() - .filter_map(|rt| { - let schema = match rt.schema { - Some(s) => match serde_json::from_str(&s) { - Ok(v) => Some(v), - Err(e) => { - println!("Warning: failed to parse schema for {}: {}", rt.name, e); - return None; - } - }, - None => None, - }; - Some(HubResourceType { - id: rt.id, - name: rt.name, - schema, - app: rt.app, - description: rt.description, - format_extension: Some(rt.format_extension), - display_name: rt.display_name, - }) - }) - .collect(); + let resource_types = hub_resource_types::fetch(DEFAULT_HUB_BASE_URL, None, None).await?; println!("Fetched {} resource types from hub", resource_types.len()); create_dir_all(&*HUB_RT_CACHE_DIR)?; - let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); + let cache_path = hub_resource_types::cache_path(); let content = serde_json::to_string_pretty(&resource_types) .with_context(|| "Failed to serialize resource types")?; @@ -516,147 +424,6 @@ async fn cache_hub_resource_types() -> anyhow::Result<()> { Ok(()) } -pub async fn sync_cached_resource_types(db: &sqlx::Pool) -> anyhow::Result<()> { - let cache_path = format!("{}/{}", *HUB_RT_CACHE_DIR, HUB_RT_CACHE_FILE); - - if tokio::fs::metadata(&cache_path).await.is_err() { - tracing::info!( - "No cached resource types found at {}, skipping sync", - cache_path - ); - return Ok(()); - } - - tracing::info!("Syncing cached resource types to admins workspace..."); - - let content = tokio::fs::read_to_string(&cache_path) - .await - .with_context(|| format!("Failed to read cache file from {}", cache_path))?; - - let cached_types: Vec = - serde_json::from_str(&content).with_context(|| "Failed to parse cached resource types")?; - - tracing::info!("Found {} cached resource types", cached_types.len()); - - // Get existing resource types in admins workspace. `format_extension` is part of - // the comparison below, so a type whose only change is gaining or losing it is - // not mistaken for unchanged; `is_fileset` decides whether it may take one. - let existing_types: Vec<( - String, - Option, - Option, - Option, - bool, - Option, - )> = sqlx::query_as( - "SELECT name, schema, description, format_extension, is_fileset, display_name FROM resource_type WHERE workspace_id = 'admins'", - ) - .fetch_all(db) - .await - .with_context(|| "Failed to fetch existing resource types")?; - - let existing_map: std::collections::HashMap< - String, - ( - Option, - Option, - Option, - bool, - Option, - ), - > = existing_types - .into_iter() - .map( - |(name, schema, desc, format_extension, is_fileset, display_name)| { - ( - name, - (schema, desc, format_extension, is_fileset, display_name), - ) - }, - ) - .collect(); - - let mut synced_count = 0; - let mut skipped_count = 0; - - for rt in cached_types { - let existing = existing_map.get(&rt.name); - let is_fileset = existing.map(|(_, _, _, f, _)| *f).unwrap_or(false); - let stored_extension = existing.and_then(|(_, _, e, _, _)| e.clone()); - let stored_display_name = existing.and_then(|(_, _, _, _, n)| n.clone()); - // A fileset is a set of files, so it cannot also be one file. Create, update - // and the manual sync all reject the pair; this writer would otherwise - // persist it onto a same-named local fileset. - // - // A cache with no key at all leaves the stored value alone, so the target is - // what is already there — which is also what makes the comparison below - // agree with the write instead of re-upserting the row on every boot. - let format_extension = if is_fileset { - None - } else { - match &rt.format_extension { - Some(from_cache) => from_cache.clone(), - None => stored_extension.clone(), - } - }; - // No key in the cache leaves the stored name alone, as for the extension. So does a name - // too long for the column: one bad entry must not fail the upsert and end the sync. - let display_name = match &rt.display_name { - Some(Some(name)) if name.chars().count() > 100 => { - tracing::warn!( - "Ignoring the display_name of resource type {}: longer than 100 characters", - rt.name - ); - stored_display_name.clone() - } - Some(from_cache) => from_cache.clone(), - None => stored_display_name.clone(), - }; - - if let Some((existing_schema, existing_desc, _, _, _)) = existing { - if existing_schema == &rt.schema - && existing_desc == &rt.description - && stored_extension == format_extension - && stored_display_name == display_name - { - skipped_count += 1; - continue; - } - } - - // Insert or update resource type - sqlx::query( - // `format_extension` is resolved above rather than coalesced here: a - // COALESCE could never clear one, so a hub that dropped an extension - // would leave the stale value behind forever. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) - VALUES ('admins', $1, $2, $3, $4, $5, now()) - ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, - format_extension = EXCLUDED.format_extension, - display_name = EXCLUDED.display_name, edited_at = now()", - ) - .bind(&rt.name) - .bind(&rt.schema) - .bind(&rt.description) - .bind(&format_extension) - .bind(&display_name) - .execute(db) - .await - .with_context(|| format!("Failed to upsert resource type {}", rt.name))?; - - synced_count += 1; - } - - tracing::info!( - "Synced {} resource types to admins workspace ({} skipped as unchanged)", - synced_count, - skipped_count - ); - - Ok(()) -} - fn print_help() { println!("Windmill - a fast, open-source workflow engine and job runner."); println!(); @@ -696,7 +463,7 @@ fn print_help() { println!(" LICENSE_KEY = None (EE only) Enterprise license key (workers require valid key)"); println!(" RUN_UPDATE_CA_CERTIFICATE_AT_START = false Run system CA update at startup"); println!(" RUN_UPDATE_CA_CERTIFICATE_PATH = /usr/sbin/update-ca-certificates Path to CA update tool"); - println!(" SYNC_CACHED_RT = false Sync cached resource types to admins workspace on server start"); + println!(" SYNC_CACHED_RT = false Sync the image's cached resource types to the admins workspace on server start"); println!(" HUB_BASE_URL = https://hub.windmill.dev Hub to fetch scripts from in `cache` mode (server/worker use the DB setting instead)"); println!(); println!("Notes:"); @@ -1015,8 +782,10 @@ async fn windmill_main() -> anyhow::Result<()> { .map(|v| v.to_lowercase() == "true" || v == "1") .unwrap_or(false) { - if let Err(e) = sync_cached_resource_types(db).await { - tracing::warn!("Failed to sync cached resource types: {:#}", e); + match hub_resource_types::sync_from_image_cache(db).await { + Ok(Some(outcome)) => tracing::info!("{}", outcome.summary()), + Ok(None) => {} + Err(e) => tracing::warn!("Failed to sync cached resource types: {:#}", e), } } } @@ -1748,6 +1517,7 @@ Windmill Community Edition {GIT_VERSION} if server_mode { if let Some(db) = conn.as_sql() { schedule_stats(&db, &HTTP_CLIENT).await; + hub_resource_types::spawn_daily_sync(db.clone(), killpill_tx.subscribe()); } } @@ -2322,6 +2092,9 @@ async fn process_notify_event( RESTART_COORDINATION_SETTING => { // Internal coordination key for staggered restarts, no action needed } + SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING => { + // Read by the daily resource type sync each time it runs + } "plain_emails_telemetry" => { let enabled = sqlx::query_scalar!( "SELECT value FROM global_settings WHERE name = 'plain_emails_telemetry'" @@ -2407,6 +2180,8 @@ pub async fn run_workers( &*PY311_CACHE_DIR, &*PY312_CACHE_DIR, &*PY313_CACHE_DIR, + &*PY314_CACHE_DIR, + &*PY315_CACHE_DIR, &*BUN_BUNDLE_CACHE_DIR, &*GO_CACHE_DIR, &*GO_BIN_CACHE_DIR, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 8d0af4915d..8de85baa24 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -4136,6 +4136,20 @@ pub async fn monitor_db( } _ => {} } + match windmill_common::runnable_job_stats::cleanup_old_runnable_job_stats( + db, + windmill_common::runnable_job_stats::RETENTION_DAYS, + ) + .await + { + Ok(count) if count > 0 => { + tracing::info!("Deleted {} old runnable job stats rows", count); + } + Err(e) => { + tracing::error!("Error cleaning up runnable job stats: {:?}", e); + } + _ => {} + } } } }; diff --git a/backend/summarized_schema.txt b/backend/summarized_schema.txt index 284a822b47..ac07dc06d6 100644 --- a/backend/summarized_schema.txt +++ b/backend/summarized_schema.txt @@ -180,6 +180,7 @@ resource_type: workspace_id(char), name(char), schema(jsonb), description(text), FK: (workspace_id) -> workspace(id) resume_job: id(uuid), job(uuid), flow(uuid), created_at(ts), value(jsonb), approver(char), resume_id(int), approved(bool) FK: (flow) -> v2_job_queue(id) +runnable_job_stats: hour(bigint), workspace_id(char), runnable_path(char), worker_group(text), job_count(int), total_duration_ms(bigint), max_memory_peak(int), sum_memory_peak(bigint), memory_sample_count(int), total_cpu_ms(bigint) runnable_settings: hash(bigint), debouncing_settings(bigint), concurrency_settings(bigint) schedule: workspace_id(char), path(char), edited_by(char), edited_at(ts), schedule(char), enabled(bool), script_path(char), args(jsonb), extra_perms(jsonb), is_flow(bool), email(char), error(text), timezone(char), on_failure(char), on_recovery(char), on_failure_times(int), on_failure_exact(bool), on_failure_extra_args(jsonb), on_recovery_times(int), on_recovery_extra_args(jsonb), ws_error_handler_muted(bool), retry(jsonb), summary(char), no_flow_overlap(bool), tag(char), paused_until(ts), on_success(char), on_success_extra_args(jsonb), cron_version(text), description(text), dynamic_skip(char), labels(text[]) FK: (workspace_id) -> workspace(id) diff --git a/backend/tests/ai_decision.rs b/backend/tests/ai_decision.rs index 04b310f2c1..d6359c1cea 100644 --- a/backend/tests/ai_decision.rs +++ b/backend/tests/ai_decision.rs @@ -117,7 +117,7 @@ async fn test_ai_decision_answers_route_a_following_branch( "expr": "results.d.output.intent.choice === 'refund'", "modules": [{"id": "r", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(choice){ return 'handled ' + choice }", "input_transforms": {"choice": { "type": "javascript", diff --git a/backend/tests/debounce_e2e.rs b/backend/tests/debounce_e2e.rs index 9a2351f135..709629d825 100644 --- a/backend/tests/debounce_e2e.rs +++ b/backend/tests/debounce_e2e.rs @@ -33,7 +33,7 @@ mod debounce_e2e { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main(items: any[]) { return items }", "input_transforms": { "items": { "type": "javascript", "expr": "flow_input.items" } diff --git a/backend/tests/drafts_delete_for_user.rs b/backend/tests/drafts_delete_for_user.rs new file mode 100644 index 0000000000..9c48f8828c --- /dev/null +++ b/backend/tests/drafts_delete_for_user.rs @@ -0,0 +1,85 @@ +//! Only a workspace admin can delete a draft that belongs to another user. +//! +//! A discard through `drafts/update` is scoped to the caller's own row, so it +//! never reaches someone else's draft. `drafts/delete_for_user` is the admin +//! route for that, and it must stay closed to everyone else. + +use serde_json::{json, Value}; +use sqlx::{Pool, Postgres}; + +use windmill_test_utils::*; + +#[sqlx::test(fixtures("base"))] +async fn test_only_admin_deletes_another_users_draft(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let base = format!("http://localhost:{port}/api/w/test-workspace/drafts"); + let client = reqwest::Client::new(); + let path = "script/u/test-user-2/theirs"; + + let save = |token: &'static str| { + client + .post(format!("{base}/update/{path}")) + .header("Authorization", format!("Bearer {token}")) + .json(&json!({ "value": { "path": "u/test-user-2/theirs", "content": "" } })) + .send() + }; + let resp = save("SECRET_TOKEN_2").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + + let delete = |token: &'static str| { + client + .post(format!( + "{base}/delete_for_user/{path}?username=test-user-2" + )) + .header("Authorization", format!("Bearer {token}")) + .send() + }; + let own_draft = || async { + client + .get(format!("{base}/get_own/{path}")) + .header("Authorization", "Bearer SECRET_TOKEN_2") + .send() + .await? + .json::() + .await + }; + + let resp = delete("SECRET_TOKEN_3").await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + assert!( + !own_draft().await?.is_null(), + "a non-admin deleted the draft" + ); + + let resp = delete("SECRET_TOKEN").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + assert!(own_draft().await?.is_null(), "the draft is still there"); + + let resp = delete("SECRET_TOKEN").await?; + assert_eq!(resp.status(), 404, "{}", resp.text().await?); + + // Without a username the delete takes every draft at the path. + for token in ["SECRET_TOKEN_2", "SECRET_TOKEN"] { + let resp = save(token).await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + } + let delete_all = |token: &'static str| { + client + .post(format!("{base}/delete_for_user/{path}")) + .header("Authorization", format!("Bearer {token}")) + .send() + }; + let resp = delete_all("SECRET_TOKEN_3").await?; + assert_eq!(resp.status(), 403, "{}", resp.text().await?); + let resp = delete_all("SECRET_TOKEN").await?; + assert_eq!(resp.status(), 200, "{}", resp.text().await?); + let left: i64 = sqlx::query_scalar( + "SELECT count(*) FROM draft WHERE workspace_id = 'test-workspace' AND path = 'u/test-user-2/theirs'", + ) + .fetch_one(&db) + .await?; + assert_eq!(left, 0, "drafts survived a path-wide delete"); + Ok(()) +} diff --git a/backend/tests/end_user_email.rs b/backend/tests/end_user_email.rs index 366c220c62..0d8ff7be9a 100644 --- a/backend/tests/end_user_email.rs +++ b/backend/tests/end_user_email.rs @@ -1,10 +1,8 @@ //! Tests for WM_END_USER_EMAIL environment variable. //! //! These tests verify that WM_END_USER_EMAIL is populated with the authenticated -//! user's email when executing app components. -//! -//! TODO: Add tests for scripts and flows once public execution endpoints are identified. -//! Currently only apps support non-workspace-member execution via OptAuthed + token lookup. +//! user's email when executing app components and when running scripts and flows directly. +//! Only apps support non-workspace-member execution via OptAuthed + token lookup. use serde_json::json; use sqlx::{Pool, Postgres}; @@ -27,40 +25,6 @@ fn authed(builder: reqwest::RequestBuilder, token: &str) -> reqwest::RequestBuil builder.header("Authorization", format!("Bearer {}", token)) } -// TODO: Script tests - need to identify public execution endpoints for non-workspace-members -// async fn run_script(port: u16, token: &str) -> anyhow::Result { -// let url = format!( -// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/p/f/test/get_end_user_email", -// port -// ); -// let resp = authed(client().post(&url), token) -// .json(&json!({})) -// .send() -// .await?; -// if !resp.status().is_success() { -// anyhow::bail!("script run failed: {} - {}", resp.status(), resp.text().await?); -// } -// Ok(resp.json::().await? -// .as_str().unwrap_or("").to_string()) -// } - -// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members -// async fn run_flow(port: u16, token: &str) -> anyhow::Result { -// let url = format!( -// "http://localhost:{}/api/w/test-workspace/jobs/run_wait_result/f/f/test/get_end_user_email_flow", -// port -// ); -// let resp = authed(client().post(&url), token) -// .json(&json!({})) -// .send() -// .await?; -// if !resp.status().is_success() { -// anyhow::bail!("flow run failed: {} - {}", resp.status(), resp.text().await?); -// } -// Ok(resp.json::().await? -// .as_str().unwrap_or("").to_string()) -// } - /// Create an app with inline script via API async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result<()> { let url = format!("http://localhost:{}/api/w/test-workspace/apps/create", port); @@ -74,8 +38,8 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result< "subgrids": {}, "hiddenInlineScripts": [{ "name": "get_email", - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "path": "f/test/email_app/get_email" }] }, @@ -89,7 +53,7 @@ async fn create_app_with_inline_script(port: u16, path: &str) -> anyhow::Result< "one_of_inputs": {} }, // SHA256 hash of raw_code content for anonymous execution - "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": { "static_inputs": {}, "one_of_inputs": {} } @@ -120,8 +84,8 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res "css": "", "inlineScripts": [{ "name": "get_email", - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }" }] }, "policy": { @@ -134,7 +98,7 @@ async fn create_raw_app_with_inline_script(port: u16, path: &str) -> anyhow::Res "one_of_inputs": {} }, // SHA256 hash of raw_code content for anonymous execution - "rawscript/6428aba5aa2d3ea8e1215bfdccbedd3718b18da7a239e3778a9787bb9a0ea606": { + "rawscript/894d2f85ea2411e777e7d5787366c9fec4c1a7a5f83482748f1d129196c9f54f": { "static_inputs": {}, "one_of_inputs": {} } @@ -233,8 +197,8 @@ async fn run_app_inline_script( "args": {}, "component": "get_email", "raw_code": { - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "path": format!("{}/get_email", app_path) } }); @@ -270,8 +234,8 @@ async fn run_raw_app_inline_script( "args": {}, "component": "get_email", "raw_code": { - "language": "deno", - "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }" + "language": "bun", + "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }" } }); if force_viewer { @@ -312,41 +276,111 @@ async fn wait_for_job_result(port: u16, token: &str, job_id: &str) -> anyhow::Re anyhow::bail!("timeout waiting for job result") } -// TODO: Script tests - need to identify public execution endpoints for non-workspace-members -// #[cfg(feature = "deno_core")] -// #[sqlx::test(fixtures("base", "end_user_email"))] -// async fn test_script_wm_end_user_email(db: Pool) -> anyhow::Result<()> { -// initialize_tracing().await; -// set_jwt_secret().await; -// let server = ApiServer::start(db.clone()).await?; -// let port = server.addr.port(); -// -// in_test_worker(Connection::Sql(db.clone()), async move { -// let result = run_script(port, SAME_WS_TOKEN).await?; -// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); -// Ok::<(), anyhow::Error>(()) -// }, port).await?; -// -// Ok(()) -// } +async fn run_direct(port: u16, token: &str, route: &str) -> anyhow::Result { + let url = format!( + "http://localhost:{}/api/w/test-workspace/jobs/run/{}", + port, route + ); + let resp = authed(client().post(&url), token) + .json(&json!({})) + .send() + .await?; + if !resp.status().is_success() { + anyhow::bail!( + "run {route} failed: {} - {}", + resp.status(), + resp.text().await? + ); + } + let job_id = resp.text().await?; + wait_for_job_result(port, token, &job_id).await +} -// TODO: Flow tests - need to identify public execution endpoints for non-workspace-members -// #[cfg(feature = "deno_core")] -// #[sqlx::test(fixtures("base", "end_user_email"))] -// async fn test_flow_wm_end_user_email(db: Pool) -> anyhow::Result<()> { -// initialize_tracing().await; -// set_jwt_secret().await; -// let server = ApiServer::start(db.clone()).await?; -// let port = server.addr.port(); -// -// in_test_worker(Connection::Sql(db.clone()), async move { -// let result = run_flow(port, SAME_WS_TOKEN).await?; -// assert_eq!(result, SAME_WS_EMAIL, "same workspace user should get their email"); -// Ok::<(), anyhow::Error>(()) -// }, port).await?; -// -// Ok(()) -// } +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_direct_run_wm_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + in_test_worker( + Connection::Sql(db.clone()), + async move { + let result = run_direct(port, SAME_WS_TOKEN, "p/f/test/get_end_user_email").await?; + assert_eq!(result, SAME_WS_EMAIL, "a script run should see its caller"); + + let result = + run_direct(port, SAME_WS_TOKEN, "f/f/test/get_end_user_email_flow").await?; + assert_eq!( + result, SAME_WS_EMAIL, + "every flow step should see the flow's caller" + ); + + Ok::<(), anyhow::Error>(()) + }, + port, + ) + .await?; + + Ok(()) +} + +/// A job's `WM_TOKEN` names the identity the job runs as, which differs from its end user when +/// the job runs on behalf of another account. A run it starts must keep the job's end user. +#[cfg(feature = "deno_core")] +#[sqlx::test(fixtures("base", "end_user_email"))] +async fn test_job_token_run_keeps_end_user_email(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + set_jwt_secret().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + let calling_job = uuid::Uuid::new_v4(); + sqlx::query( + "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id, end_user_email) + VALUES ($1, 'test2@windmill.dev', 'test-user-2', false, false, '{}', '{}', 'test-workspace', $2)", + ) + .bind(calling_job) + .bind(SAME_WS_EMAIL) + .execute(&db) + .await?; + let wm_token = windmill_common::auth::create_jwt_token( + windmill_common::db::Authed { + email: "test2@windmill.dev".to_string(), + username: "test-user-2".to_string(), + is_admin: false, + is_operator: false, + groups: vec!["all".to_string()], + folders: vec![], + scopes: None, + token_prefix: None, + }, + "test-workspace", + 3600, + Some(calling_job), + None, + None, + None, + ) + .await?; + + in_test_worker( + Connection::Sql(db.clone()), + async move { + let result = run_direct(port, &wm_token, "p/f/test/get_end_user_email").await?; + assert_eq!( + result, SAME_WS_EMAIL, + "the calling job's end user, not its run-as identity" + ); + Ok::<(), anyhow::Error>(()) + }, + port, + ) + .await?; + + Ok(()) +} #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base", "end_user_email"))] diff --git a/backend/tests/error_handler.rs b/backend/tests/error_handler.rs index 4f58685221..2e10006d2f 100644 --- a/backend/tests/error_handler.rs +++ b/backend/tests/error_handler.rs @@ -104,7 +104,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'deno', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', '') + VALUES ('test-workspace', 1111111111, 'f/test/error_handler', $1, 'bun', 'script', 'test-user', '{}', 'Error handler script', 'Handles failed job completions', E'{}\n//bun.lock\n') "#, error_handler_code ) @@ -118,7 +118,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'deno', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', '') + VALUES ('test-workspace', $1, 'f/test/failing_script', $2, 'bun', 'script', 'test-user', '{}', 'Failing test script', 'A script that always fails', E'{}\n//bun.lock\n') "#, failing_script_hash, failing_script_code @@ -155,7 +155,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), @@ -234,7 +234,7 @@ async fn test_error_handler_muted_on_script(db: Pool) -> anyhow::Resul sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return "handled"; }', 'deno', 'script', 'test-user', '{}', '', '', '') + VALUES ('test-workspace', 3333333333, 'f/test/error_handler', 'export function main() { return "handled"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n') "#, ) .execute(&db) @@ -245,7 +245,7 @@ async fn test_error_handler_muted_on_script(db: Pool) -> anyhow::Resul sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock, ws_error_handler_muted) - VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error("fail"); }', 'deno', 'script', 'test-user', '{}', '', '', '', true) + VALUES ('test-workspace', $1, 'f/test/muted_failing_script', 'export function main() { throw new Error("fail"); }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n', true) "#, failing_script_hash, ) @@ -280,7 +280,7 @@ async fn test_error_handler_muted_on_script(db: Pool) -> anyhow::Resul cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), @@ -331,7 +331,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return "handled"; }', 'deno', 'script', 'test-user', '{}', '', '', '') + VALUES ('test-workspace', 5555555555, 'f/test/error_handler', 'export function main() { return "handled"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n') "#, ) .execute(&db) @@ -342,7 +342,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return "ok"; }', 'deno', 'script', 'test-user', '{}', '', '', '') + VALUES ('test-workspace', $1, 'f/test/success_script', 'export function main() { return "ok"; }', 'bun', 'script', 'test-user', '{}', '', '', E'{}\n//bun.lock\n') "#, success_script_hash, ) @@ -377,7 +377,7 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), @@ -411,3 +411,70 @@ async fn test_error_handler_not_triggered_on_success(db: Pool) -> anyh Ok(()) } + +/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job, +/// and its failure must reach the workspace error handler like a workspace script's. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_error_handler_triggered_on_hub_script_failure( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::jobs::JobPayload; + + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + + let version = "990000002"; + let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR; + tokio::fs::create_dir_all(hub_dir).await?; + tokio::fs::write( + format!("{hub_dir}/{version}"), + r#"{"content":"exit 1","lockfile":null,"language":"bash","schema":{},"summary":null}"#, + ) + .await?; + + // The workspace handler settings are cached per workspace id across the tests of this + // binary, so this reuses the handler path every other test here configures. + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', 1111111112, 'f/test/error_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')", + ) + .execute(&db) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET error_handler = '{"path": "script/f/test/error_handler"}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let completed_job = RunJob::from(JobPayload::ScriptHub { + path: format!("hub/{version}/test/fail"), + apply_preprocessor: false, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!(!completed_job.success, "Hub script should have failed"); + + let mut handler_parent = None; + for _ in 0..50 { + handler_parent = sqlx::query_scalar::<_, Option>( + "SELECT parent_job FROM v2_job + WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'error_handler@windmill.dev'", + ) + .fetch_optional(&db) + .await?; + if handler_parent.is_some() { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + assert_eq!( + handler_parent, + Some(Some(completed_job.id)), + "Workspace error handler should handle the failed hub script" + ); + + Ok(()) +} diff --git a/backend/tests/fixtures/end_user_email.sql b/backend/tests/fixtures/end_user_email.sql index 596d34e3e3..3b376525a4 100644 --- a/backend/tests/fixtures/end_user_email.sql +++ b/backend/tests/fixtures/end_user_email.sql @@ -40,9 +40,9 @@ VALUES (encode(sha256('NO_WS_TOKEN'::bytea), 'hex'), 'NO_WS_TOKE', 'NO_WS_TOKEN' INSERT INTO script (workspace_id, created_by, content, schema, summary, description, path, hash, language, lock, kind, extra_perms) VALUES ( 'test-workspace', 'test-user', - 'export function main() { return Deno.env.get("WM_END_USER_EMAIL") || ""; }', + 'export function main() { return process.env.WM_END_USER_EMAIL || ""; }', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'deno', '', 'script', + 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email', 900001, 'bun', E'{}\n//bun.lock\n', 'script', '{"g/all": true}' ); @@ -51,7 +51,7 @@ INSERT INTO flow (workspace_id, summary, description, path, versions, schema, va VALUES ( 'test-workspace', 'Returns WM_END_USER_EMAIL', '', 'f/test/get_end_user_email_flow', '{900002}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}]}', 'test-user', '{"g/all": true}' ); @@ -60,6 +60,6 @@ INSERT INTO flow_version (id, workspace_id, path, schema, value, created_by) VALUES ( 900002, 'test-workspace', 'f/test/get_end_user_email_flow', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{},"required":[],"type":"object"}', - '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "deno", "content": "export function main() { return Deno.env.get(\"WM_END_USER_EMAIL\") || \"\"; }", "input_transforms": {}}}]}', + '{"modules": [{"id": "a", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}, {"id": "b", "value": {"type": "rawscript", "language": "bun", "content": "export function main() { return process.env.WM_END_USER_EMAIL || \"\"; }", "input_transforms": {}}}]}', 'test-user' ); diff --git a/backend/tests/fixtures/hello.sql b/backend/tests/fixtures/hello.sql index 962f6441d9..4c40ac3290 100644 --- a/backend/tests/fixtures/hello.sql +++ b/backend/tests/fixtures/hello.sql @@ -11,7 +11,7 @@ export function main(world: string) { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}', '', '', -'f/system/hello', 123412, 'deno', ''); +'f/system/hello', 123412, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -28,7 +28,7 @@ export async function main(foo: string, bar: string) { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"default":null,"description":"","originalType":"string","type":"string"},"bar":{"default":null,"description":"","originalType":"string","type":"string"}},"required":["foo","bar"],"type":"object"}', '', '', -'f/system/hello_with_preprocessor', 123413, 'deno', ''); +'f/system/hello_with_preprocessor', 123413, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -140,7 +140,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche 'f/system/hello_with_nodes_flow', '{1443253234253454}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}', -E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', +E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"bun","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"bun","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', 'system' ); @@ -149,7 +149,7 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b 'test-workspace', 'f/system/hello_with_nodes_flow', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"world":{"default":"world","description":"","type":"string"}},"required":[],"type":"object"}', -E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"deno","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"deno","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', +E'{"modules":[{"id":"a","value":{"type":"forloopflow","modules":[{"id":"b","value":{"type":"rawscript","content":"export function main(world: string) {\\n const greet = `Hello ${world}!`;\\n console.log(greet)\\n return greet\\n}\\n","language":"bun","input_transforms":{"world":{"type":"javascript","expr":"flow_input.iter.value"}},"is_trigger":false}},{"id":"c","value":{"type":"rawscript","content":"export function main(hello: string) {\\n const dareyou = `Did you just say \\"${hello}\\"??!`;\\n console.log(dareyou)\\n return dareyou\\n}","language":"bun","input_transforms":{"hello":{"type":"javascript","value":"${results.b}","expr":"`${results.b}`"}},"is_trigger":false}}],"iterator":{"type":"javascript","expr":"[''foo'', ''bar'', ''baz'']"},"skip_failures":true,"parallel":false}}],"same_worker":false}', 'system' ); @@ -160,7 +160,7 @@ INSERT INTO public.flow(workspace_id, summary, description, path, versions, sche 'f/system/hello_with_preprocessor', '{1443253234253456}', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}', -E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', +E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"bun","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"bun","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', 'system' ); @@ -169,6 +169,6 @@ INSERT INTO public.flow_version(id, workspace_id, path, schema, value, created_b 'test-workspace', 'f/system/hello_with_preprocessor', '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"foo":{"type":"string","description":"","default":""},"bar":{"type":"string","description":"","default":""}},"required":[],"type":"object","order":["foo","bar"]}', -E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"deno","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"deno","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', +E'{"modules":[{"id":"a","value":{"type":"rawscript","content":"export async function main(foo: string, bar: string) {\\n return ''Hello '' + foo + ''-'' + bar\\n}\\n","language":"bun","is_trigger":false,"input_transforms":{"bar":{"expr":"flow_input.bar","type":"javascript"},"foo":{"expr":"flow_input.foo","type":"javascript"}}}}],"preprocessor_module":{"id":"preprocessor","value":{"type":"rawscript","content":"export async function preprocessor(\\n\\twm_trigger: {\\n\\t\\tkind: ''http'' | ''email'' | ''webhook'' | ''websocket'' | ''kafka'' | ''nats'',\\n\\t\\thttp?: {\\n\\t\\t\\troute: string\\n\\t\\t\\tpath: string\\n\\t\\t\\tmethod: string\\n\\t\\t\\tparams: Record\\n\\t\\t\\tquery: Record\\n\\t\\t\\theaders: Record\\n\\t\\t},\\n\\t\\twebsocket?: {\\n\\t\\t\\turl: string\\n\\t\\t},\\n\\t\\tkafka?: {\\n\\t\\t\\tbrokers: string[]\\n\\t\\t\\ttopic: string\\n\\t\\t\\tgroup_id: string\\n\\t\\t},\\n\\t\\tnats?: {\\n\\t\\t\\tservers: string[]\\n\\t\\t\\tsubject: string\\n\\t\\t\\theaders?: Record\\n\\t\\t\\tstatus?: number\\n\\t\\t\\tdescription?: string\\n\\t\\t\\tlength: number\\n\\t\\t}\\n\\t},\\n) {\\n\\treturn {\\n\\t\\tfoo: ''bar'',\\n\\t\\tbar: ''baz''\\n\\t}\\n}","language":"bun","is_trigger":false,"input_transforms":{"wm_trigger":{"type":"static"}}}}}', 'system' ); diff --git a/backend/tests/fixtures/jobs_read_auth.sql b/backend/tests/fixtures/jobs_read_auth.sql index e32b777a74..e175a9c065 100644 --- a/backend/tests/fixtures/jobs_read_auth.sql +++ b/backend/tests/fixtures/jobs_read_auth.sql @@ -98,6 +98,32 @@ INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, resu ('18181818-1818-1818-1818-181818181818', 'test-workspace', 1000, 'success'::job_status, '{"agent": "AGENT_CHAT_RESULT"}'); +-- A lock job for flow `f/shared/flow1`, as `wmill generate-metadata` queues it, and the +-- token CI would lock with. +INSERT INTO token(token_hash, token_prefix, token, email, label, super_admin, scopes) VALUES ( + encode(sha256('RUN_SCOPED_DEPS_TOKEN'::bytea), 'hex'), 'RUN_DEPS_', 'RUN_SCOPED_DEPS_TOKEN', + 'test2@windmill.dev', 'lock token', false, + ARRAY['jobs:run:dependencies:f/shared/*'] +); +INSERT INTO public.v2_job ( + id, workspace_id, created_by, created_at, permissioned_as, permissioned_as_email, + kind, runnable_path, tag, visible_to_owner +) VALUES ( + '19191919-1919-1919-1919-191919191919', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'flowdependencies', 'f/shared/flow1', 'dependency', true +), ( + -- Queued by a broader token with `*` as its path, which the request body allows. + '20202020-2020-2020-2020-202020202020', 'test-workspace', 'test-user-2', + '2023-01-01 00:00:00', 'u/test-user-2', 'test2@windmill.dev', + 'dependencies', '*', 'dependency', true +); +INSERT INTO public.v2_job_completed (id, workspace_id, duration_ms, status, result) VALUES + ('19191919-1919-1919-1919-191919191919', 'test-workspace', 1000, 'success'::job_status, + '{"lock": "DEPS_RESULT"}'), + ('20202020-2020-2020-2020-202020202020', 'test-workspace', 1000, 'success'::job_status, + '{"lock": "WILDCARD_DEPS_RESULT"}'); + -- A token pairing an app scope with a run scope, as someone driving an app's components -- programmatically would build. `APP_INLINE_JOB` is an inline-script component run: no -- `jobs:run` scope can name its kind, so only the `apps:run` half puts it in reach. diff --git a/backend/tests/fixtures/schedule.sql b/backend/tests/fixtures/schedule.sql index 8d851369b0..999aa4770f 100644 --- a/backend/tests/fixtures/schedule.sql +++ b/backend/tests/fixtures/schedule.sql @@ -13,7 +13,7 @@ export async function main(fail: boolean = true) { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"fail":{"default":true,"description":"","type":"boolean"}},"required":[],"type":"object"}', '', '', -'f/system/failing_script', 12349, 'deno', ''); +'f/system/failing_script', 12349, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -26,7 +26,7 @@ export async function main() { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"error":{"default":null,"description":"","properties":{},"type":"object"}},"required":["path","schedule_path","error"],"type":"object"}', '', '', -'f/system/schedule_error_handler', 123410, 'deno', ''); +'f/system/schedule_error_handler', 123410, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.script(workspace_id, created_by, content, schema, summary, description, path, hash, language, lock) VALUES ( 'test-workspace', @@ -39,7 +39,7 @@ export async function main() { '{"$schema":"https://json-schema.org/draft/2020-12/schema","properties":{"path":{"default":null,"description":"","type":"string"},"schedule_path":{"default":null,"description":"","type":"string"},"previous_job_error":{"default":null,"description":"","type":"string"},"result":{"default":null,"description":"","type":"string"}},"required":["path","schedule_path","previous_job_error","result"],"type":"object"}', '', '', -'f/system/schedule_recovery_handler', 123411, 'deno', ''); +'f/system/schedule_recovery_handler', 123411, 'bun', E'{}\n//bun.lock\n'); INSERT INTO public.flow(workspace_id, summary, description, path, versions, schema, value, edited_by) VALUES ( 'test-workspace', diff --git a/backend/tests/flow_engine_parity.rs b/backend/tests/flow_engine_parity.rs index 3806bb2886..2d4f72f336 100644 --- a/backend/tests/flow_engine_parity.rs +++ b/backend/tests/flow_engine_parity.rs @@ -85,7 +85,7 @@ async fn test_flow_linear_input_transforms(db: Pool) -> anyhow::Result modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: [static_input("x", 10), static_input("y", 5)].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(x: number, y: number) { return {sum: x + y, product: x * y, items: [1, 2, 3, 4, 5]}; @@ -105,7 +105,7 @@ export function main(x: number, y: number) { js_input("filtered", "results.a.items.filter(x => x > 2)"), js_input("from_flow_input", "flow_input.multiplier * results.a.sum"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(total: number, doubled_items: number[], filtered: number[], from_flow_input: number) { return {total, doubled_items, filtered, from_flow_input}; @@ -157,7 +157,7 @@ async fn test_flow_forloop_complex_expressions(db: Pool) -> anyhow::Re "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -199,7 +199,7 @@ export function main() { js_input("index", "flow_input.iter.index"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(user_name: string, user_score: number, bonus: number, index: number) { return {name: user_name, final_score: user_score + bonus, position: index}; @@ -261,7 +261,7 @@ async fn test_flow_branchone_conditions(db: Pool) -> anyhow::Result<() "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {status: "premium", score: 95, items: [1, 2, 3]}; @@ -293,7 +293,7 @@ export function main() { js_input("score_from_a", "results.a.score"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(discount: number, score_from_a: number) { return {branch: "premium_high", discount, score_from_a}; @@ -320,7 +320,7 @@ export function main(discount: number, score_from_a: number) { "premium_low", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {branch: "premium_low", discount: 10}; @@ -344,7 +344,7 @@ export function main() { "default_branch", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {branch: "default", discount: 0}; @@ -398,7 +398,7 @@ async fn test_flow_previous_result_aggregation(db: Pool) -> anyhow::Re "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {value: 10, items: [1, 2, 3]}; @@ -424,7 +424,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(prev_value: number, prev_items_sum: number) { return {value: prev_value * 2, sum: prev_items_sum}; @@ -452,7 +452,7 @@ export function main(prev_value: number, prev_items_sum: number) { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(a_value: number, b_value: number, b_sum: number, combined: number) { return {a_value, b_value, b_sum, combined}; @@ -504,7 +504,7 @@ async fn test_flow_nested_complexity(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -541,7 +541,7 @@ export function main() { js_input("values", "results.data.base_values.map(v => v * flow_input.iter.value.multiplier)"), js_input("sum", "results.data.base_values.reduce((a, b) => a + b, 0) * flow_input.iter.value.multiplier"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(cat_name: string, multiplier: number, values: number[], sum: number) { return {category: cat_name, multiplier, computed_values: values, total: sum}; @@ -602,7 +602,7 @@ async fn test_flow_object_transformations(db: Pool) -> anyhow::Result< modules: vec![ flow_module("source", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -633,7 +633,7 @@ export function main() { js_input("active_bonus", "results.source.config.activeBonus"), js_input("all_users", "results.source.users"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(admins: string[], active_count: number, admin_bonus: number, active_bonus: number, all_users: any[]) { // Compute user_summary and total_bonus in the script since complex expressions in input_transforms @@ -702,7 +702,7 @@ async fn test_flow_skip_if_expressions(db: Pool) -> anyhow::Result<()> "check", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {should_skip: true, value: 100}; @@ -722,7 +722,7 @@ export function main() { "maybe_skipped", FlowModuleValue::RawScript { input_transforms: [js_input("input_val", "results.check.value * 2")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(input_val: number) { return {processed: input_val, was_run: true}; @@ -750,7 +750,7 @@ export function main(input_val: number) { js_input("prev", "previous_result"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(check_val: number, prev: any) { return {check_val, previous: prev}; @@ -799,7 +799,7 @@ async fn test_flow_template_literals(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -825,7 +825,7 @@ export function main() { js_input("upper_name", "results.data.firstName.toUpperCase()"), js_input("items_formatted", "`Items: ${results.data.items.map(i => i.charAt(0).toUpperCase() + i.slice(1)).join(', ')}`"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(full_name: string, greeting: string, items_str: string, upper_name: string, items_formatted: string) { return {full_name, greeting, items_str, upper_name, items_formatted}; @@ -873,7 +873,7 @@ async fn test_flow_optional_chaining(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -905,7 +905,7 @@ export function main() { js_input("default_city", "results.data.partial_user?.address?.city ?? 'Unknown'"), js_input("default_country", "results.data.user?.address?.country ?? 'USA'"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(city: string, missing_city: any, null_user_name: any, default_city: string, default_country: string) { return {city, missing_city, null_user_name, default_city, default_country}; @@ -955,7 +955,7 @@ async fn test_flow_parallel_forloop(db: Pool) -> anyhow::Result<()> { "data", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {items: [1, 2, 3, 4, 5]}; @@ -990,7 +990,7 @@ export function main() { js_input("cubed", "flow_input.iter.value.value ** 3"), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(original: number, squared: number, cubed: number) { return {original, squared, cubed}; @@ -1089,7 +1089,7 @@ async fn test_flow_env_access(db: Pool) -> anyhow::Result<()> { js_input("retry_count", "flow_env.CONFIG.retries"), js_input("has_auth", "flow_env.CONFIG.features.includes('auth')"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(env_name: string, is_debug: boolean, timeout_val: number, api_url: string, retry_count: number, has_auth: boolean) { return {env_name, is_debug, timeout_val, api_url, retry_count, has_auth}; @@ -1158,7 +1158,7 @@ async fn test_flow_input_and_env_combined(db: Pool) -> anyhow::Result< // Conditional based on both js_input("multiplier", "flow_env.ENV === 'production' ? flow_input.prod_mult : 1"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(effective_limit: number, env_prefix: string, is_prod: boolean, doubled_input: number, multiplier: number) { return {effective_limit, env_prefix, is_prod, doubled_input, final_value: doubled_input * multiplier}; @@ -1215,7 +1215,7 @@ async fn test_flow_results_optional_chaining(db: Pool) -> anyhow::Resu "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1287,7 +1287,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( user_name: string, @@ -1361,7 +1361,7 @@ async fn test_flow_large_integers(db: Pool) -> anyhow::Result<()> { "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1401,7 +1401,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( small: number, i32_max: number, over_i32: number, @@ -1459,7 +1459,7 @@ async fn test_flow_unicode_emoji(db: Pool) -> anyhow::Result<()> { modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1489,7 +1489,7 @@ export function main() { js_input("mixed_upper", "results.a.mixed.toUpperCase()"), js_input("template", "`Welcome: ${results.a.greeting}`"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( greeting: string, greeting_len: number, simple_str: string, has_world: boolean, @@ -1544,7 +1544,7 @@ async fn test_flow_complex_array_operations(db: Pool) -> anyhow::Resul modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1585,7 +1585,7 @@ export function main() { js_input("first_three", "results.a.numbers.slice(0, 3)"), js_input("last_two", "results.a.numbers.slice(-2)"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( sorted_asc: number[], sorted_desc: number[], active_names: string[], @@ -1651,7 +1651,7 @@ async fn test_flow_multiline_expressions(db: Pool) -> anyhow::Result<( modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1710,7 +1710,7 @@ export function main() { } "#), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( subtotal: number, discounted_total: number, @@ -1768,7 +1768,7 @@ async fn test_flow_spread_with_results(db: Pool) -> anyhow::Result<()> modules: vec![ flow_module("a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1803,7 +1803,7 @@ export function main() { // Combining multiple spreads js_input("combined", "{config: {...results.a.config}, tags: [...results.a.tags], source: 'flow'}"), ].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( merged_config: any, all_tags: string[], full_user: any, @@ -1869,7 +1869,7 @@ async fn test_flow_nested_forloop_results_access(db: Pool) -> anyhow:: "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { @@ -1908,7 +1908,7 @@ export function main() { js_input("multiplier", "results.a.multiplier"), // Access outer step from inside for-loop ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(category: string, multiplier: number) { return { @@ -1939,7 +1939,7 @@ export function main(category: string, multiplier: number) { js_input("original_mult", "results.a.multiplier"), // Access outer step ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(items: any[], category: string, original_mult: number) { return { @@ -2013,7 +2013,7 @@ async fn test_flow_results_non_existent_step(db: Pool) -> anyhow::Resu "a", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { value: 42 }; @@ -2050,7 +2050,7 @@ export function main() { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main( existing: number, @@ -2126,7 +2126,7 @@ async fn test_flow_env_skip_if_inside_branchone(db: Pool) -> anyhow::R "inner", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {ran: true}; @@ -2156,7 +2156,7 @@ export function main() { "branch_marker", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {marker: "branch-marker"}; @@ -2193,7 +2193,7 @@ export function main() { "after", FlowModuleValue::RawScript { input_transforms: [js_input("prev", "previous_result")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(prev: any) { return {prev}; @@ -2258,7 +2258,7 @@ async fn test_flow_env_skip_if_nested_branchone(db: Pool) -> anyhow::R "leaf", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {ran: true}; @@ -2282,7 +2282,7 @@ export function main() { "inner_marker", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {marker: "inner"}; @@ -2337,7 +2337,7 @@ export function main() { "after", FlowModuleValue::RawScript { input_transforms: [js_input("prev", "previous_result")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(prev: any) { return {prev}; @@ -2414,7 +2414,7 @@ async fn test_flow_env_complex_input_transform_in_branch(db: Pool) -> "Math.min(flow_env.LIMIT, 10) + flow_env.OFFSET", )] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(value: number) { return {value}; @@ -2487,7 +2487,7 @@ async fn test_flow_env_skip_if_in_parallel_forloop(db: Pool) -> anyhow "marker", FlowModuleValue::RawScript { input_transforms: [js_input("i", "flow_input.iter.value")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(i: number) { return {marker: i}; @@ -2508,7 +2508,7 @@ export function main(i: number) { "leaf", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {ran: true}; @@ -2594,7 +2594,7 @@ async fn test_flow_env_imported_flow_uses_own_env(db: Pool) -> anyhow: "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(key: string) { return {key}; }", "input_transforms": { "key": { "type": "javascript", "expr": "flow_env.KEY" } @@ -2679,7 +2679,7 @@ async fn test_flow_env_imported_flow_inherits_when_unset(db: Pool) -> "id": "marker", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {marker: \"from-imported\"}; }", "input_transforms": {} } @@ -2688,7 +2688,7 @@ async fn test_flow_env_imported_flow_inherits_when_unset(db: Pool) -> "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {ran: true}; }", "input_transforms": {} }, @@ -2791,7 +2791,7 @@ async fn test_flow_env_imported_flow_with_nested_branch(db: Pool) -> a "id": "marker", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {marker: \"from-imported-branch\"}; }", "input_transforms": {} } @@ -2800,7 +2800,7 @@ async fn test_flow_env_imported_flow_with_nested_branch(db: Pool) -> a "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return {ran: true}; }", "input_transforms": {} }, @@ -2932,13 +2932,13 @@ async fn test_flow_env_marks_sub_flows_only_without_ancestor_env( "parallel": true, "modules": [{ "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(key: any) { return key ?? null; }", "input_transforms": { "key": { "type": "javascript", "expr": "flow_env.KEY" } } }}, { "id": "tail", "value": { // A single simple step is inlined rather than stored as a flow node. "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(prev: any) { return prev; }", "input_transforms": { "prev": { "type": "javascript", "expr": "results.leaf" } } }}] @@ -3024,7 +3024,7 @@ async fn test_flow_env_in_stop_after_if(db: Pool) -> anyhow::Result<() "first", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {stage: "first"}; @@ -3052,7 +3052,7 @@ export function main() { "second", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {stage: "second"}; @@ -3110,7 +3110,7 @@ async fn test_stop_after_if_error_include_result(db: Pool) -> anyhow:: "step", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return { userErrors: ["email taken"], ok: false }; @@ -3200,7 +3200,7 @@ async fn test_flow_env_in_retry_if(db: Pool) -> anyhow::Result<()> { "fails", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { throw new Error("nope"); @@ -3274,7 +3274,7 @@ async fn test_flow_env_in_stop_after_all_iters_if(db: Pool) -> anyhow: "iter_step", FlowModuleValue::RawScript { input_transforms: [js_input("i", "flow_input.iter.value")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(i: number) { return {iter: i}; @@ -3316,7 +3316,7 @@ export function main(i: number) { "after", FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main() { return {stage: "after-loop"}; @@ -3375,7 +3375,7 @@ async fn test_stop_after_all_iters_if_error_includes_result( "iter_step", FlowModuleValue::RawScript { input_transforms: [js_input("i", "flow_input.iter.value")].into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#" export function main(i: number) { return { iter: i }; @@ -3453,8 +3453,8 @@ async fn test_flow_expr_step_tag(db: Pool) -> anyhow::Result<()> { id, FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, - content: "export function main() { return { lang: 'bun' } }".to_string(), + language: ScriptLang::Bun, + content: "export function main() { return { lang: 'python3' } }".to_string(), path: None, lock: None, tag: tag.map(str::to_string), @@ -3484,7 +3484,7 @@ async fn test_flow_expr_step_tag(db: Pool) -> anyhow::Result<()> { .bind(job.id) .fetch_one(&db) .await?; - assert_eq!(b_tag, "bun"); + assert_eq!(b_tag, "python3"); assert!(!job.success); let result = job.json_result().unwrap(); diff --git a/backend/tests/job_payload.rs b/backend/tests/job_payload.rs index 0038effc78..c40feb1650 100644 --- a/backend/tests/job_payload.rs +++ b/backend/tests/job_payload.rs @@ -54,7 +54,7 @@ mod job_payload { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, job_token_scopes: None, @@ -85,7 +85,7 @@ mod job_payload { cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: true, concurrency_settings: @@ -165,7 +165,7 @@ mod job_payload { let test = || async { let result = RunJob::from(JobPayload::FlowScript { id: flow_scripts[0], - language: ScriptLang::Deno, + language: ScriptLang::Bun, concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), cache_ttl: None, @@ -185,7 +185,7 @@ mod job_payload { let test = || async { let result = RunJob::from(JobPayload::FlowScript { id: flow_scripts[1], - language: ScriptLang::Deno, + language: ScriptLang::Bun, concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default(), cache_ttl: None, @@ -263,7 +263,7 @@ mod job_payload { let result = RunJob::from(JobPayload::Dependencies { path: "f/system/hello".to_string(), hash: ScriptHash(123412), - language: ScriptLang::Deno, + language: ScriptLang::Bun, debouncing_settings: Default::default(), dedicated_worker: None, }) @@ -351,7 +351,7 @@ mod job_payload { console.log(greet) return greet }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -407,7 +407,7 @@ mod job_payload { return greet }"# .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, }) .run_until_complete(&db, false, port) .await @@ -416,7 +416,10 @@ mod job_payload { assert_eq!( result, - json!({ "lock": "", "status": "Successful lock file generation" }) + json!({ + "lock": "{\n \"dependencies\": {}\n}\n//bun.lock\n", + "status": "Successful lock file generation" + }) ); }; test_for_versions(VERSION_FLAGS.iter().copied(), test).await; @@ -626,7 +629,7 @@ mod job_payload { console.log(greet) return greet }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -673,7 +676,7 @@ mod job_payload { "content": r#"export function main(world: string) { return `Hello ${world}!`; }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -685,7 +688,7 @@ mod job_payload { "content": r#"export function main(world: string, a: string) { return `${a} ${world}!`; }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" }, "a": { "type": "javascript", "expr": "results.a" } @@ -704,7 +707,7 @@ mod job_payload { "x": { "type": "javascript", "expr": "flow_input.iter.value" } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": r#"export function main(world: string, b: string, x: string) { return `${x}: ${b} ${world}!`; }"#, @@ -857,7 +860,7 @@ mod job_payload { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } }, @@ -874,7 +877,7 @@ mod job_payload { "id": "inner_first", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } }, @@ -884,7 +887,7 @@ mod job_payload { "id": "inner_second", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" }, "first": { "type": "javascript", "expr": "results.inner_first" } @@ -1045,7 +1048,7 @@ mod job_payload { "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "iter_val": { "type": "javascript", "expr": "flow_input.iter.value" }, "tag": { "type": "javascript", "expr": "flow_input.tag" } @@ -1244,7 +1247,7 @@ mod job_payload { "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "tag": { "type": "javascript", "expr": "flow_input.tag" }, "iv": { "type": "javascript", "expr": "flow_input.iter.value" } @@ -1388,7 +1391,7 @@ mod job_payload { "id": "leaf", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "tag": { "type": "javascript", "expr": "flow_input.tag" } }, diff --git a/backend/tests/jobs_read_auth.rs b/backend/tests/jobs_read_auth.rs index 6a101200a1..bffacfbaf0 100644 --- a/backend/tests/jobs_read_auth.rs +++ b/backend/tests/jobs_read_auth.rs @@ -476,6 +476,44 @@ async fn test_single_job_read_authorization(db: Pool) -> anyhow::Resul ); } + // A lock job belongs to `jobs:run:dependencies`: its path is whatever the caller sent, + // so a token scoped to run the flow at that path must not read it. + let path = "completed/get_result/19191919-1919-1919-1919-191919191919"; + let (status, body) = get(&base, path, Some("RUN_SCOPED_DEPS_TOKEN")).await; + assert!( + status.is_success() && body.contains("DEPS_RESULT"), + "dependencies-scoped token must read its lock job (got {status}): {body}" + ); + let (status, body) = get(&base, path, Some("RUN_SCOPED_TOKEN")).await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "a flow-scoped token must not read a lock job (got {status}): {body}" + ); + let (status, body) = get( + &base, + &format!("completed/get_result/{FLOW_JOB}"), + Some("RUN_SCOPED_DEPS_TOKEN"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "dependencies-scoped token must not read a flow run at its path (got {status}): {body}" + ); + // A lock job filed at `*` matches no path-scoped grant: the stored path is not a pattern. + let (status, body) = get( + &base, + "completed/get_result/20202020-2020-2020-2020-202020202020", + Some("RUN_SCOPED_DEPS_TOKEN"), + ) + .await; + assert_eq!( + status, + reqwest::StatusCode::NOT_FOUND, + "a path-scoped token must not read a lock job filed at `*` (got {status}): {body}" + ); + // An `apps:run:` scope is a start grant too: the inline-script component run it // launched — a kind no `jobs:run` scope can name — stays readable to a token scoped // to that app, and stays out of reach for one that is only scoped to run jobs. diff --git a/backend/tests/otel.rs b/backend/tests/otel.rs index f1e042037f..30585c08a1 100644 --- a/backend/tests/otel.rs +++ b/backend/tests/otel.rs @@ -176,6 +176,10 @@ async fn test_all_metrics_e2e() { otel_set_worker_uptime("w-uptime", 3600.0); otel_set_health_status_phase("healthy"); otel_set_health_db_unresponsive(true); + otel_set_worker_memory("w-res", "gpu", Some(512), Some(128), Some(1024)); + otel_add_worker_cpu("w-res", "gpu", 1.5, 40, 7, 0.35); + otel_set_worker_occupancy_rate("w-res", "gpu", "5m", 0.25); + otel_record_job_memory_peak("python3", 100 * 1024 * 1024); // ── Histograms ────────────────────────────────────────────────── @@ -189,7 +193,7 @@ async fn test_all_metrics_e2e() { let metrics = flush_and_get_metrics(&state); let names = metric_names(&metrics); - // ── Verify all 21 metric names are present ────────────────────── + // ── Verify all metric names are present ───────────────────────── let expected = [ "windmill.queue.push_count", @@ -213,6 +217,15 @@ async fn test_all_metrics_e2e() { "windmill.worker.uptime", "windmill.health.status", "windmill.health.db_unresponsive", + "windmill.worker.memory_usage", + "windmill.worker.windmill_memory_usage", + "windmill.worker.memory_limit", + "windmill.worker.cpu_usage", + "windmill.worker.cpu_periods", + "windmill.worker.cpu_throttled_periods", + "windmill.worker.cpu_throttled_time", + "windmill.worker.occupancy_rate", + "windmill.worker.job_memory_peak", ]; for name in expected { assert!( @@ -290,6 +303,18 @@ async fn test_all_metrics_e2e() { .expect("worker.busy data point with worker=worker-test-1 not found"); assert_eq!(dp.1, 1); + let m = find_metric(&metrics, "windmill.worker.memory_limit").unwrap(); + let values = gauge_i64_values(m); + let dp = values + .iter() + .find(|(attrs, _)| { + attrs + .iter() + .any(|kv| kv.key.as_str() == "worker_group" && kv.value.as_str() == "gpu") + }) + .expect("worker.memory_limit data point with worker_group=gpu not found"); + assert_eq!(dp.1, 1024); + let m = find_metric(&metrics, "windmill.db.pool.active").unwrap(); assert_eq!(gauge_i64_values(m)[0].1, 5); let m = find_metric(&metrics, "windmill.db.pool.idle").unwrap(); diff --git a/backend/tests/restart_at_step_api.rs b/backend/tests/restart_at_step_api.rs index 2b57de2c64..f6fc2a04d3 100644 --- a/backend/tests/restart_at_step_api.rs +++ b/backend/tests/restart_at_step_api.rs @@ -217,7 +217,7 @@ async fn test_api_restart_at_step_rejects_parallel_loop(db: Pool) -> a "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": {}, "content": "export function main() { return 'ok' }" } diff --git a/backend/tests/retry.rs b/backend/tests/retry.rs index e804b96f0e..d80e386b61 100644 --- a/backend/tests/retry.rs +++ b/backend/tests/retry.rs @@ -70,12 +70,20 @@ mod retry { fn inner_step() -> &'static str { r#" +import { connect } from "node:net"; + export async function main(index, port) { - const buf = new Uint8Array([0]); - const sock = await Deno.connect({ port }); - await sock.write(new Uint8Array([index])); - if (await sock.read(buf) != 1) throw Error("read"); - return buf[0]; + return await new Promise((resolve, reject) => { + const sock = connect({ port, host: "127.0.0.1" }, () => { + sock.write(new Uint8Array([index])); + }); + sock.on("data", (data) => { + resolve(data[0]); + sock.destroy(); + }); + sock.on("error", reject); + sock.on("close", () => reject(Error("read"))); + }); } "# } @@ -105,7 +113,7 @@ def main(last, port): "port": { "type": "javascript", "expr": "flow_input.port" }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": inner_step(), }, }], @@ -281,7 +289,7 @@ def main(last, port): "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return \"ok\" }", }, }, { @@ -293,7 +301,7 @@ def main(last, port): "port": { "type": "javascript", "expr": "flow_input.port" }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": inner_step(), }, "retry": { "constant": { "attempts": 1, "seconds": 0 } }, diff --git a/backend/tests/success_handler.rs b/backend/tests/success_handler.rs index 8fc4182d9e..f087090992 100644 --- a/backend/tests/success_handler.rs +++ b/backend/tests/success_handler.rs @@ -123,7 +123,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'deno', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', '') + VALUES ('test-workspace', 1234567890, 'f/test/success_handler', $1, 'bun', 'script', 'test-user', '{}', 'Success handler script', 'Handles successful job completions', E'{}\n//bun.lock\n') "#, success_handler_code ) @@ -138,7 +138,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: sqlx::query!( r#" INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) - VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'deno', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', '') + VALUES ('test-workspace', $1, 'f/test/simple_script', $2, 'bun', 'script', 'test-user', '{}', 'Simple test script', 'A simple test script', E'{}\n//bun.lock\n') "#, test_script_hash, test_script_code @@ -175,7 +175,7 @@ export async function main(path: string, email: string, job_id: string, is_flow: cache_ttl: None, cache_ignore_s3_path: None, dedicated_worker: None, - language: ScriptLang::Deno, + language: ScriptLang::Bun, priority: None, apply_preprocessor: false, concurrency_settings: ConcurrencySettings::default(), @@ -239,3 +239,70 @@ export async function main(path: string, email: string, job_id: string, is_flow: Ok(()) } + +/// A hub script run at the top level (a schedule, a trigger, the API) is a `script_hub` job, +/// and its success must reach the workspace success handler like a workspace script's. +#[cfg(all(feature = "enterprise", feature = "private"))] +#[sqlx::test(fixtures("base"))] +async fn test_success_handler_triggered_on_hub_script_success( + db: Pool, +) -> anyhow::Result<()> { + use windmill_common::jobs::JobPayload; + + initialize_tracing().await; + + let server = ApiServer::start(db.clone()).await?; + + let version = "990000003"; + let hub_dir = &*windmill_common::worker::HUB_CACHE_DIR; + tokio::fs::create_dir_all(hub_dir).await?; + tokio::fs::write( + format!("{hub_dir}/{version}"), + r#"{"content":"echo ok","lockfile":null,"language":"bash","schema":{},"summary":null}"#, + ) + .await?; + + // The workspace handler settings are cached per workspace id across the tests of this + // binary, so this reuses the handler path every other test here configures. + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, summary, description, lock) + VALUES ('test-workspace', 1234567891, 'f/test/success_handler', 'echo handled', 'bash', 'script', 'test-user', '{}', '', '', '')", + ) + .execute(&db) + .await?; + sqlx::query( + r#"UPDATE workspace_settings SET success_handler = '{"path": "script/f/test/success_handler"}'::jsonb + WHERE workspace_id = 'test-workspace'"#, + ) + .execute(&db) + .await?; + + let completed_job = RunJob::from(JobPayload::ScriptHub { + path: format!("hub/{version}/test/ok"), + apply_preprocessor: false, + }) + .run_until_complete(&db, false, server.addr.port()) + .await; + assert!(completed_job.success, "Hub script should have succeeded"); + + let mut handler_parent = None; + for _ in 0..50 { + handler_parent = sqlx::query_scalar::<_, Option>( + "SELECT parent_job FROM v2_job + WHERE workspace_id = 'test-workspace' AND permissioned_as_email = 'success_handler@windmill.dev'", + ) + .fetch_optional(&db) + .await?; + if handler_parent.is_some() { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + } + assert_eq!( + handler_parent, + Some(Some(completed_job.id)), + "Workspace success handler should handle the successful hub script" + ); + + Ok(()) +} diff --git a/backend/tests/suspend_resume.rs b/backend/tests/suspend_resume.rs index 6fb5d68004..0decc16519 100644 --- a/backend/tests/suspend_resume.rs +++ b/backend/tests/suspend_resume.rs @@ -63,11 +63,11 @@ mod suspend_resume { "op": { "type": "javascript", "expr": "flow_input.op ?? 'resume'", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "\ export async function main(n, port, op) {\ - const job = Deno.env.get('WM_JOB_ID'); - const token = Deno.env.get('WM_TOKEN'); + const job = process.env.WM_JOB_ID; + const token = process.env.WM_TOKEN; const r = await fetch( `http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\ {\ @@ -102,7 +102,7 @@ mod suspend_resume { "resumes": { "type": "javascript", "expr": "resumes", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, resume, resumes) { return { n: n + 1, resume, resumes } }" }, "suspend": { @@ -116,7 +116,7 @@ mod suspend_resume { "resumes": { "type": "javascript", "expr": "resumes", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(last, resume, resumes) { return { last, resume, resumes } }" }, }], @@ -256,11 +256,11 @@ mod suspend_resume { "port": { "type": "javascript", "expr": "flow_input.port" }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "\ export async function main(port) {\ - const job = Deno.env.get('WM_JOB_ID');\ - const token = Deno.env.get('WM_TOKEN');\ + const job = process.env.WM_JOB_ID;\ + const token = process.env.WM_TOKEN;\ const secret = await (await fetch(\ `http://localhost:${port}/api/w/test-workspace/jobs/job_signature/${job}/0?token=${token}&approver=ruben`,\ { headers: { 'Authorization': `Bearer ${token}` } }\ @@ -278,7 +278,7 @@ mod suspend_resume { "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'b ran' }", }, // The gate holds `b` back, so `b` never runs and its error policy describes @@ -290,7 +290,7 @@ mod suspend_resume { "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'handled' }", }, }, @@ -343,7 +343,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -355,7 +355,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -454,7 +454,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -466,7 +466,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -556,7 +556,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -567,7 +567,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -658,7 +658,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -670,7 +670,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -764,7 +764,7 @@ mod suspend_resume { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step1'; }" }, "suspend": { @@ -776,7 +776,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 'step2 - after approval'; }" } }] @@ -933,7 +933,7 @@ mod suspend_resume { "suspend": { "required_events": 1, "timeout": 86400 }, "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { return 1 }", "input_transforms": {}, }, @@ -942,7 +942,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { return 42 }", "input_transforms": {}, }, @@ -980,7 +980,7 @@ mod suspend_resume { "suspend": { "required_events": APPROVALS, "timeout": 86400 }, "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { return 1 }", "input_transforms": {}, }, @@ -989,7 +989,7 @@ mod suspend_resume { "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(resumes) { return resumes.length }", "input_transforms": { "resumes": { "type": "javascript", "expr": "resumes" }, diff --git a/backend/tests/wac_suspend_started_at.rs b/backend/tests/wac_suspend_started_at.rs index 5a83730d24..ca364e5993 100644 --- a/backend/tests/wac_suspend_started_at.rs +++ b/backend/tests/wac_suspend_started_at.rs @@ -109,3 +109,91 @@ async fn wac_suspend_stands_down_for_a_cancel(db: Pool) -> anyhow::Res Ok(()) } + +/// Each park clears `started_at`, so the completion would otherwise store only the last +/// segment: a workflow that waited an hour on its tasks would read as a sub-second job that +/// started an hour late. The completed row spans the workflow from its first start, and +/// keeps the last segment aside: that is what is metered and what worker-time readers sum. +#[sqlx::test] +async fn wac_completion_spans_the_workflow_and_keeps_the_last_segment( + db: Pool, +) -> anyhow::Result<()> { + let job_id = Uuid::new_v4(); + sqlx::query( + "INSERT INTO v2_job (id, workspace_id, created_by, created_at, permissioned_as, \ + permissioned_as_email, kind, script_lang, runnable_path, tag, visible_to_owner) \ + VALUES ($1, 'test-workspace', 'test-user', now() - interval '2 hours', 'u/test-user', \ + 'test@windmill.dev', 'script', 'bun', 'u/test-user/wac', 'bun', true)", + ) + .bind(job_id) + .execute(&db) + .await?; + sqlx::query( + "INSERT INTO v2_job_queue (id, workspace_id, scheduled_for, running, tag, started_at) \ + VALUES ($1, 'test-workspace', now(), true, 'bun', now() - interval '1 hour')", + ) + .bind(job_id) + .execute(&db) + .await?; + + // Two parks, each followed by the pull re-stamping `started_at`: the second park must + // not replace the first start with its own segment's. + for resumed_ago in ["10 minutes", "3 seconds"] { + let mut tx = db.begin().await?; + suspend_wac_parent(&mut tx, &job_id, "test-workspace", 1, 3600.0).await?; + tx.commit().await?; + sqlx::query( + "UPDATE v2_job_queue SET suspend = 0, started_at = now() - $2::text::interval \ + WHERE id = $1", + ) + .bind(job_id) + .bind(resumed_ago) + .execute(&db) + .await?; + } + + let job = windmill_queue::get_mini_completed_job(&job_id, "test-workspace", &db) + .await? + .unwrap(); + let result = serde_json::value::to_raw_value(&serde_json::json!("done"))?; + let (_, span) = windmill_queue::add_completed_job( + &db, + &job, + true, + false, + sqlx::types::Json(&result), + None, + 0, + None, + false, + Some(3000), + false, + ) + .await?; + + let (started_ago_s, duration_ms, last_segment_ms): (f64, i64, Option) = sqlx::query_as( + "SELECT extract(epoch FROM now() - started_at)::float8, duration_ms, \ + (extras->>'wac_last_segment_ms')::bigint \ + FROM v2_job_completed WHERE id = $1", + ) + .bind(job_id) + .fetch_one(&db) + .await?; + assert!( + (started_ago_s - 3600.0).abs() < 60.0, + "started_at must be the first start, got {started_ago_s}s ago" + ); + assert!( + (duration_ms - 3_600_000).abs() < 60_000, + "duration_ms must be the wall time since the first start, got {duration_ms}" + ); + assert_eq!(last_segment_ms, Some(3000)); + assert_eq!( + span.duration_ms, duration_ms, + "a flow records the step with the span the row holds" + ); + assert!(span.outlasts_run(job.started_at)); + assert_eq!(span.last_run_ms, 3000, "only the last segment is left to meter"); + + Ok(()) +} diff --git a/backend/tests/wm_token_confinement.rs b/backend/tests/wm_token_confinement.rs index 23d88301b7..e75170ec6f 100644 --- a/backend/tests/wm_token_confinement.rs +++ b/backend/tests/wm_token_confinement.rs @@ -319,7 +319,7 @@ async fn test_wm_token_is_confined_to_its_workspace(db: Pool) -> anyho ); } // ...and the `settings/global` keys on the allowlist, which the CLI reads from a job: on a - // git-sync push, and in `u/admin/hub_sync`. `ws_base_url` is the control: the handler + // git-sync push, and in `wmill hub pull`. `ws_base_url` is the control: the handler // leaves it as ungated as these, so only the allowlist stops it. for key in ["automate_username_creation", "uid", "hub_base_url"] { let resp = authed( diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 73d3fc2567..751752eb00 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -166,7 +166,7 @@ async fn test_iteration_parallel(db: Pool) -> anyhow::Result<()> { #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] -async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { +async fn test_bun_flow(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; @@ -183,7 +183,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { id: "a".to_string(), value: FlowModuleValue::RawScript { input_transforms: Default::default(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: numbers.to_string(), path: None, lock: None, @@ -232,7 +232,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { }, )] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: doubles.to_string(), path: None, lock: None, @@ -296,7 +296,7 @@ async fn test_deno_flow(db: Pool) -> anyhow::Result<()> { let port = server.addr.port(); for i in 0..50 { - println!("deno flow iteration: {}", i); + println!("bun flow iteration: {}", i); let job = run_job_in_new_worker_until_complete(&db, false, job.clone(), port).await; // println!("job: {:#?}", job.flow_status); let result = job.json_result().unwrap(); @@ -346,13 +346,13 @@ async fn test_identity(db: Pool) -> anyhow::Result<()> { #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] -async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { +async fn test_bun_flow_same_worker(db: Pool) -> anyhow::Result<()> { initialize_tracing().await; let server = ApiServer::start(db.clone()).await?; let write_file = r#"export async function main(loop: boolean, i: number, path: string) { - await Deno.writeTextFile(`./shared/${path}`, `${loop} ${i}`); + await Bun.write(`./shared/${path}`, `${loop} ${i}`); }"# .to_string(); @@ -373,7 +373,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: write_file.clone(), path: None, lock: None, @@ -432,7 +432,7 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: write_file, path: None, lock: None, @@ -472,9 +472,9 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { InputTransform::Static { value: windmill_common::worker::to_raw_value(&"outer.txt") }, )] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#"export async function main(path: string, path2: string) { - return await Deno.readTextFile(`./shared/${path}`) + "," + await Deno.readTextFile(`./shared/${path2}`); + return await Bun.file(`./shared/${path}`).text() + "," + await Bun.file(`./shared/${path2}`).text(); }"# .to_string(), path: None, @@ -546,9 +546,9 @@ async fn test_deno_flow_same_worker(db: Pool) -> anyhow::Result<()> { ), ] .into(), - language: ScriptLang::Deno, + language: ScriptLang::Bun, content: r#"export async function main(path: string, loops: string[], path2: string) { - return await Deno.readTextFile(`./shared/${path}`) + "," + loops + "," + await Deno.readTextFile(`./shared/${path2}`); + return await Bun.file(`./shared/${path}`).text() + "," + loops + "," + await Bun.file(`./shared/${path2}`).text(); }"# .to_string(), path: None, @@ -618,7 +618,7 @@ async fn test_same_worker_survives_empty_branch(db: Pool) -> anyhow::R "id": "c", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return 1 }", } }] @@ -630,7 +630,7 @@ async fn test_same_worker_survives_empty_branch(db: Pool) -> anyhow::R "id": "b", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return 42 }", } } @@ -671,7 +671,7 @@ async fn test_flow_result_by_id(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return 42 }", } }, @@ -686,7 +686,7 @@ async fn test_flow_result_by_id(db: Pool) -> anyhow::Result<()> { "value": { "input_transforms": {"v": {"type": "javascript", "expr": "results.a"}}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(v){ return v }", } @@ -3857,7 +3857,7 @@ fn module_add_item_to_list(i: i32, id: &str) -> serde_json::Value { } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(array, i){ array.push(i); return array }", } }) @@ -3868,7 +3868,7 @@ fn module_failure() -> serde_json::Value { "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ throw Error('failure') }", } }) @@ -3886,7 +3886,7 @@ async fn test_branchone_simple(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -3924,7 +3924,7 @@ async fn test_branchone_with_cond(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -3962,7 +3962,7 @@ async fn test_branchall_sequential(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -4002,7 +4002,7 @@ async fn test_branchall_simple(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -4050,7 +4050,7 @@ async fn test_branchall_skip_failure(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -4086,7 +4086,7 @@ async fn test_branchall_skip_failure(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -4132,7 +4132,7 @@ async fn test_branchone_nested(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [] }", } }, @@ -4191,7 +4191,7 @@ async fn test_branchall_nested(db: Pool) -> anyhow::Result<()> { "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(){ return [1] }", } }, @@ -4257,7 +4257,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "n": { "type": "javascript", "expr": "flow_input.n", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, l) { if (n == 0) throw Error(JSON.stringify(l)); return { l: [...l, 0] } }", }, }, { @@ -4268,7 +4268,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "n": { "type": "javascript", "expr": "flow_input.n", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, l) { if (n == 1) throw Error(JSON.stringify(l)); return { l: [...l, 1] } }", }, }, { @@ -4278,7 +4278,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "n": { "type": "javascript", "expr": "flow_input.n", }, }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(n, l) { if (n == 2) throw Error(JSON.stringify(l)); return { l: [...l, 2] } }", }, }], @@ -4286,7 +4286,7 @@ async fn test_failure_module(db: Pool) -> anyhow::Result<()> { "value": { "input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(error) { return { 'from failure module': error } }", } }, @@ -4475,7 +4475,7 @@ fn hanging_step_value() -> serde_json::Value { serde_json::json!({ "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export async function main() { await new Promise((r) => setTimeout(r, 600000)); }", }) } @@ -4487,7 +4487,7 @@ fn marker_failure_module() -> serde_json::Value { "value": { "input_transforms": { "error": { "type": "javascript", "expr": "previous_result", } }, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main(error) { return { handled_unrecoverable: true, error } }", } }) @@ -4585,7 +4585,7 @@ async fn test_unrecoverable_failure_on_continue_on_error_runs_failure_module( "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return { ran_b: true }; }", }, }, @@ -4624,7 +4624,7 @@ async fn test_run_wait_result_early_return_with_failure_module( "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": {}, "content": "export function main() { throw new Error('boom'); }", }, @@ -4632,7 +4632,7 @@ async fn test_run_wait_result_early_return_with_failure_module( "failure_module": { "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": {}, "content": "export function main() { return { recovered: true } }", }, @@ -4720,7 +4720,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow:: "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { throw new Error('boom'); }", }, }], @@ -4734,7 +4734,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow:: "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return { ran_b: true } }", }, }], @@ -4742,7 +4742,7 @@ async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow:: "value": { "input_transforms": {}, "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return { handled: true, recover: true } }", }, }, @@ -4797,14 +4797,23 @@ async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow:: server.close().await.unwrap(); #[cfg(feature = "quickjs")] - assert!(!chaining_error.success, "an error no handler recovered must keep the flow failed"); + assert!( + !chaining_error.success, + "an error no handler recovered must keep the flow failed" + ); - assert!(stopped.success, "a recovered failure inside a loop should end the flow as a success"); + assert!( + stopped.success, + "a recovered failure inside a loop should end the flow as a success" + ); assert!( stopped.json_result().unwrap().get("ran_b").is_none(), "the step after the loop must not run, as without recovery" ); - assert_eq!(stopped_handler_runs, 1, "the loop must stop at the first failed iteration"); + assert_eq!( + stopped_handler_runs, 1, + "the loop must stop at the first failed iteration" + ); assert!(skipping.success); assert_eq!( @@ -4815,7 +4824,11 @@ async fn test_failure_module_recover_inside_loop(db: Pool) -> anyhow:: assert_eq!(skipping_handler_runs, 3); assert!( - !parallel.json_result().unwrap().to_string().contains("ran_b"), + !parallel + .json_result() + .unwrap() + .to_string() + .contains("ran_b"), "the parallel loop fails as without recovery, so the step after it must not run" ); Ok(()) @@ -4987,6 +5000,101 @@ async fn test_flow_lock_all(db: Pool) -> anyhow::Result<()> { Ok(()) } +/// A deployed flow runs a multi-step loop body as the flow node its deploy stored, which +/// carries its own copy of the flow's `preserve_step_tags`. A preview never loads that node. +#[sqlx::test(fixtures("base"))] +async fn test_saved_flow_preserves_nested_step_tags(db: Pool) -> anyhow::Result<()> { + use futures::StreamExt; + + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let client = reqwest::Client::new(); + let flow = json!({ + "path": "g/all/saved_step_tags", "summary": "", "description": "", "tag": "python3", + "schema": { "type": "object", "properties": {}, "required": [] }, + "value": { + "preserve_step_tags": true, + "modules": [{ + "id": "loop", + "value": { + "type": "forloopflow", "skip_failures": false, + "iterator": { "type": "static", "value": [2, 3] }, + "modules": [{ + "id": "own_tag", + "value": { + "type": "rawscript", "language": "bun", "lock": "", "tag": "bun", + "content": "export function main(n: number) { return n * n; }", + "input_transforms": { + "n": { "type": "javascript", "expr": "flow_input.iter.value" } + } + } + }, { + "id": "untagged", + "value": { + "type": "rawscript", "language": "bun", "lock": "", + "content": "export function main(n: number) { return n + 1; }", + "input_transforms": { + "n": { "type": "javascript", "expr": "results.own_tag" } + } + } + }] + } + }] + } + }); + let mut completed = listen_for_completed_jobs(&db).await; + client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/flows/create" + )) + .bearer_auth("SECRET_TOKEN") + .json(&flow) + .send() + .await? + .error_for_status()?; + let deployment = in_test_worker(&db, completed.next(), port).await.unwrap(); + let deployed = completed_job(deployment, &db).await; + assert!(deployed.success, "{:?}", deployed.result); + + let job_id = client + .post(format!( + "http://localhost:{port}/api/w/test-workspace/jobs/run/f/g/all/saved_step_tags" + )) + .bearer_auth("SECRET_TOKEN") + .json(&json!({})) + .send() + .await? + .error_for_status()? + .text() + .await? + .parse::()?; + in_test_worker(&db, completed.find(&job_id), port).await; + let job = completed_job(job_id, &db).await; + assert!(job.success, "{:?}", job.result); + let tags = sqlx::query_as::<_, (String, String)>( + "WITH RECURSIVE descendants AS ( + SELECT id FROM v2_job WHERE id = $1 + UNION ALL + SELECT child.id FROM v2_job child JOIN descendants ON child.parent_job = descendants.id + ) + SELECT DISTINCT flow_step_id, tag FROM v2_job + WHERE id IN (SELECT id FROM descendants) AND flow_step_id IN ('own_tag', 'untagged') + ORDER BY flow_step_id, tag", + ) + .bind(job_id) + .fetch_all(&db) + .await?; + assert_eq!( + tags, + vec![ + ("own_tag".to_owned(), "bun".to_owned()), + ("untagged".to_owned(), "python3".to_owned()) + ] + ); + Ok(()) +} + #[cfg(feature = "deno_core")] #[sqlx::test(fixtures("base"))] @@ -5687,7 +5795,7 @@ async fn test_job_labels(db: Pool) -> anyhow::Result<()> { console.log(greet) return { greet, wm_labels: ["yolo", "greet", "greet", world] }; }"#, - "language": "deno", + "language": "bun", "input_transforms": { "world": { "type": "javascript", "expr": "flow_input.world" } } @@ -5896,7 +6004,7 @@ async fn test_flow_substep_tag_availability_check(db: Pool) -> anyhow: "id": "a", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return 42; }", "tag": "restricted-tag", }, @@ -5956,7 +6064,7 @@ async fn test_flow_substep_tag_checked_on_resolved_value(db: Pool) -> "id": id, "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "content": "export function main() { return { lang: 'bun' } }", "tag": tag, }, diff --git a/backend/tests/zombie_flow_recovery.rs b/backend/tests/zombie_flow_recovery.rs index fb658dfb03..333aac70a8 100644 --- a/backend/tests/zombie_flow_recovery.rs +++ b/backend/tests/zombie_flow_recovery.rs @@ -78,7 +78,7 @@ async fn test_between_steps_zombie_restart_reuses_all_children( "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -91,7 +91,7 @@ async fn test_between_steps_zombie_restart_reuses_all_children( "id": "after", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, @@ -210,7 +210,7 @@ async fn test_mid_iteration_zombie_not_reused(db: Pool) -> anyhow::Res "id": "inner", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -223,7 +223,7 @@ async fn test_mid_iteration_zombie_not_reused(db: Pool) -> anyhow::Res "id": "after", "value": { "type": "rawscript", - "language": "deno", + "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, @@ -331,7 +331,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse( { "id": "inner_first", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": {}, "content": "export function main() { return 'first' }" } @@ -339,7 +339,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse( { "id": "inner_second", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "first": { "type": "javascript", "expr": "results.inner_first" } }, @@ -353,7 +353,7 @@ async fn test_nested_restart_not_swallowed_by_zombie_reuse( { "id": "after", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "b": { "type": "javascript", "expr": "results.branch" } }, @@ -466,7 +466,7 @@ async fn test_raw_flow_restart_does_not_reuse_edited_step( "modules": [{ "id": "inner", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -478,7 +478,7 @@ async fn test_raw_flow_restart_does_not_reuse_edited_step( { "id": "after", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, @@ -578,7 +578,7 @@ async fn test_non_monitor_cancel_is_not_reused(db: Pool) -> anyhow::Re "modules": [{ "id": "inner", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "v": { "type": "javascript", "expr": "flow_input.iter.value" } }, @@ -590,7 +590,7 @@ async fn test_non_monitor_cancel_is_not_reused(db: Pool) -> anyhow::Re { "id": "after", "value": { - "type": "rawscript", "language": "deno", + "type": "rawscript", "language": "bun", "input_transforms": { "loop_res": { "type": "javascript", "expr": "results.fanout" } }, diff --git a/backend/windmill-ai/Cargo.toml b/backend/windmill-ai/Cargo.toml index b1dea3ae54..44c4ac83c7 100644 --- a/backend/windmill-ai/Cargo.toml +++ b/backend/windmill-ai/Cargo.toml @@ -28,6 +28,7 @@ eventsource-stream.workspace = true futures.workspace = true http.workspace = true mime_guess.workspace = true +rand.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/backend/windmill-ai/src/ai_bedrock.rs b/backend/windmill-ai/src/ai_bedrock.rs index a9318a1f3a..f55fcca415 100644 --- a/backend/windmill-ai/src/ai_bedrock.rs +++ b/backend/windmill-ai/src/ai_bedrock.rs @@ -477,6 +477,29 @@ pub async fn assume_role_with_oidc_token( // Error Formatting // ============================================================================ +/// The error for a Bedrock stream that failed after it opened, retrying the exceptions the +/// Vercel AI SDK retries (`amazon-bedrock-stream-error.ts`) and a broken connection. The +/// SDK's own retries cover the opening call alone. +pub fn bedrock_stream_error( + error: &aws_sdk_bedrockruntime::error::SdkError< + aws_sdk_bedrockruntime::types::error::ConverseStreamOutputError, + R, + >, +) -> Error { + let message = format!("Bedrock stream error: {}", format_bedrock_error(error)); + match error.as_service_error() { + Some(e) + if !(e.is_internal_server_exception() + || e.is_model_stream_error_exception() + || e.is_service_unavailable_exception() + || e.is_throttling_exception()) => + { + Error::AIError(message) + } + _ => crate::retry::transient_error(message, None), + } +} + /// Format AWS SDK errors with detailed information pub fn format_bedrock_error(error: &aws_sdk_bedrockruntime::error::SdkError) -> String where diff --git a/backend/windmill-ai/src/ai_google.rs b/backend/windmill-ai/src/ai_google.rs index bb8ae83e13..9f48edca8a 100644 --- a/backend/windmill-ai/src/ai_google.rs +++ b/backend/windmill-ai/src/ai_google.rs @@ -326,6 +326,17 @@ pub struct GeminiSSEEvent { pub candidates: Option>, #[serde(rename = "usageMetadata")] pub usage_metadata: Option, + #[serde(default)] + pub error: Option, + #[serde(rename = "promptFeedback", default)] + pub prompt_feedback: Option, +} + +/// Set when Gemini refuses the prompt itself, in which case no candidate follows. +#[derive(Deserialize, Debug)] +pub struct GeminiPromptFeedback { + #[serde(rename = "blockReason", default)] + pub block_reason: Option, } // ============================================================================ @@ -550,10 +561,40 @@ pub fn openai_tools_to_gemini( } } +/// Follows a Gemini stream to tell a complete turn from one cut short. +#[derive(Default)] +pub struct GeminiTurn { + ended: bool, + has_output: bool, +} + +impl GeminiTurn { + pub fn observe(&mut self, event: &GeminiParsedEvent) { + // A function call arrives whole and may come without a `finishReason`, so it + // counts too, as in gemini-cli's stream validation. + self.ended |= event.finish_reason.is_some() || !event.tool_calls.is_empty(); + self.has_output |= event.text.is_some() || event.reasoning.is_some(); + } + + /// Whether the turn's final event arrived. + pub fn ended(&self) -> bool { + self.ended + } + + /// Whether a stream that closed cleanly here holds the whole turn. Gemini 2.5 closes + /// its stream without a single event when thinking spends the whole + /// `maxOutputTokens`: an empty answer, which a retry would only get again, rather + /// than a cut one. + pub fn complete_at_close(&self) -> bool { + self.ended || !self.has_output + } +} + /// Parse one Gemini SSE data line into a [`GeminiParsedEvent`]. /// /// Returns `Ok(None)` for empty data or unrecognised payloads (e.g. `"[DONE]"`). /// Logs a warning and returns `Ok(None)` on JSON parse errors rather than propagating. +/// An error the stream reports is returned as one. pub fn parse_gemini_sse_event(data: &str) -> Result, Error> { if data.is_empty() || data == "[DONE]" { return Ok(None); @@ -567,6 +608,21 @@ pub fn parse_gemini_sse_event(data: &str) -> Result, E } }; + if let Some(error) = event.error { + return Err(error.into_error("Gemini")); + } + + // `BLOCK_REASON_UNSPECIFIED` is the protobuf enum's placeholder, not a block. + if let Some(reason) = event + .prompt_feedback + .and_then(|f| f.block_reason) + .filter(|r| !matches!(r.as_str(), "" | "BLOCK_REASON_UNSPECIFIED")) + { + return Err(Error::AIError(format!( + "Gemini blocked the prompt: {reason}" + ))); + } + let mut parsed = GeminiParsedEvent { usage: event.usage_metadata, ..Default::default() }; let Some(candidates) = event.candidates else { diff --git a/backend/windmill-ai/src/lib.rs b/backend/windmill-ai/src/lib.rs index ccc40da432..634614a3cf 100644 --- a/backend/windmill-ai/src/lib.rs +++ b/backend/windmill-ai/src/lib.rs @@ -11,6 +11,7 @@ pub mod model_context; pub mod providers; pub mod proxy; pub mod query_builder; +pub mod retry; pub mod sse; pub mod types; pub mod utils; diff --git a/backend/windmill-ai/src/providers/bedrock.rs b/backend/windmill-ai/src/providers/bedrock.rs index 836244a666..5aa142a127 100644 --- a/backend/windmill-ai/src/providers/bedrock.rs +++ b/backend/windmill-ai/src/providers/bedrock.rs @@ -9,11 +9,12 @@ use super::{anthropic_model_rejects_sampling_params, REASONING_OFF_SENTINEL}; use crate::{ ai_bedrock::{ - bedrock_model_supports_prompt_caching, bedrock_stream_event_is_block_stop, - bedrock_stream_event_to_reasoning_delta, bedrock_stream_event_to_text, - bedrock_stream_event_to_tool_delta, bedrock_stream_event_to_tool_delta_with_block_index, - bedrock_stream_event_to_tool_start, bedrock_stream_event_to_tool_start_with_block_index, - build_tool_config, create_inference_config, format_bedrock_error, json_to_document, + bedrock_model_supports_prompt_caching, bedrock_stream_error, + bedrock_stream_event_is_block_stop, bedrock_stream_event_to_reasoning_delta, + bedrock_stream_event_to_text, bedrock_stream_event_to_tool_delta, + bedrock_stream_event_to_tool_delta_with_block_index, bedrock_stream_event_to_tool_start, + bedrock_stream_event_to_tool_start_with_block_index, build_tool_config, + create_inference_config, format_bedrock_error, json_to_document, openai_messages_to_bedrock, streaming_tool_calls_to_openai, BearerTokenProvider, BedrockClient, StreamingToolCall, }, @@ -24,6 +25,7 @@ use crate::{ image_handler::prepare_messages_for_api, proxy::ProxyBuildArgs, query_builder::{ParsedResponse, StreamEventSink}, + retry::{openai_stream_error_chunk, truncated_stream_error}, types::{OpenAIMessage, StreamingEvent, TokenUsage, ToolDef}, }; use bytes::Bytes; @@ -461,26 +463,31 @@ pub fn sdk_stream_to_sse( async_stream::stream! { let mut stream = stream; let mut state = BedrockSseStreamState::new(id, model, created); + let mut message_stopped = false; - loop { + let failure = loop { match stream.recv().await { Ok(Some(event)) => { + if matches!( + event, + aws_sdk_bedrockruntime::types::ConverseStreamOutput::MessageStop(_) + ) { + message_stopped = true; + } for chunk in bedrock_sse_chunks_for_event(&event, &mut state) { yield Ok(chunk); } } - Ok(None) => break, - Err(e) => { - yield Err(std::io::Error::new( - std::io::ErrorKind::Other, - e.to_string(), - )); - break; - } + Ok(None) => break (!message_stopped).then(truncated_stream_error), + Err(_) if message_stopped => break None, + Err(e) => break Some(bedrock_stream_error(&e)), } - } + }; - yield Ok(Bytes::from("data: [DONE]\n\n")); + match failure { + Some(error) => yield Ok(Bytes::from(openai_stream_error_chunk(&error))), + None => yield Ok(Bytes::from("data: [DONE]\n\n")), + } } } @@ -1081,11 +1088,19 @@ impl BedrockQueryBuilder { // Claude reasoning block for the turn (only populated when thinking is on), // attached to the first tool call for replay before toolUse. let mut reasoning: Option = None; + let mut message_stopped = false; // Process stream events using shared parsing functions loop { match stream.recv().await { Ok(Some(event)) => { + if matches!( + event, + aws_sdk_bedrockruntime::types::ConverseStreamOutput::MessageStop(_) + ) { + message_stopped = true; + } + // Fold reasoning deltas into the turn's reasoning block (for // replay before toolUse, required by Claude when thinking is // on) and stream the readable summary as a thinking affordance. @@ -1161,11 +1176,13 @@ impl BedrockQueryBuilder { } } Ok(None) => break, - Err(e) => { - return Err(Error::internal_err(format!("Bedrock stream error: {}", e))); - } + Err(_) if message_stopped => break, + Err(e) => return Err(bedrock_stream_error(&e)), } } + if !message_stopped { + return Err(truncated_stream_error()); + } // Send tool call events to stream processor if let Some(processor) = stream_event_sink.as_ref() { diff --git a/backend/windmill-ai/src/providers/google_ai.rs b/backend/windmill-ai/src/providers/google_ai.rs index 97d0383ba4..bbf45e5e57 100644 --- a/backend/windmill-ai/src/providers/google_ai.rs +++ b/backend/windmill-ai/src/providers/google_ai.rs @@ -5,17 +5,20 @@ use crate::{ parse_gemini_response, parse_gemini_sse_event, sanitize_schema_for_google, GeminiFunctionDeclaration, GeminiGenerationConfig, GeminiImageContent, GeminiImageRequest, GeminiImageResponse, GeminiInlineData, GeminiPart, GeminiPredictContent, GeminiTextRequest, - GeminiThinkingConfig, GeminiTool, + GeminiThinkingConfig, GeminiTool, GeminiTurn, }, image_handler::{download_and_encode_s3_image, prepare_messages_for_api}, proxy::{ProxyBuildArgs, ProxyRequest}, query_builder::{BuildRequestArgs, ParsedResponse, QueryBuilder, StreamEventSink}, + retry::{ + openai_stream_error_chunk, send_with_retries, transient_error, truncated_stream_error, + }, sse::{GeminiSSEParser, SSEParser}, types::*, }; use async_trait::async_trait; use bytes::Bytes; -use eventsource_stream::Eventsource; +use eventsource_stream::{EventStreamError, Eventsource}; use futures::{stream::BoxStream, StreamExt}; use http::{header, HeaderMap, HeaderValue, Method, StatusCode}; use serde::Deserialize; @@ -336,9 +339,7 @@ pub async fn handle_google_ai_chat_proxy( ) -> Result { let GoogleAIProxyRequest { request, model, stream } = build_google_ai_chat_proxy_request(args)?; - let response = - send_google_ai_proxy_request(client, request, "Failed to send request to Gemini API") - .await?; + let response = send_google_ai_proxy_request(client, request).await?; if stream { Ok(convert_streaming_response(response, &model)) @@ -356,8 +357,7 @@ pub async fn handle_google_ai_models_proxy( args: &ProxyBuildArgs<'_>, ) -> Result { let request = build_google_ai_models_proxy_request(args); - let response = - send_google_ai_proxy_request(client, request, "Failed to fetch Gemini models").await?; + let response = send_google_ai_proxy_request(client, request).await?; let gemini_resp: GeminiModelsResponse = response.json().await.map_err(|e| { Error::internal_err(format!("Failed to parse Gemini models response: {}", e)) @@ -505,26 +505,13 @@ fn add_google_ai_auth_header(headers: &mut Vec<(String, String)>, api_key: &str, async fn send_google_ai_proxy_request( client: &reqwest::Client, proxy_request: ProxyRequest, - send_error_message: &str, ) -> Result { let mut request = client.request(proxy_request.method.clone(), &proxy_request.url); for (header_name, header_value) in &proxy_request.headers { request = request.header(header_name.as_str(), header_value.as_str()); } - let response = request - .body(proxy_request.body) - .send() - .await - .map_err(|e| Error::internal_err(format!("{}: {}", send_error_message, e)))?; - - if let Err(e) = response.error_for_status_ref() { - let status = e.status().map(|s| s.to_string()).unwrap_or_default(); - let body = response.text().await.unwrap_or_default(); - return Err(Error::AIError(format!("{}: {}", status, body))); - } - - Ok(response) + send_with_retries(request.body(proxy_request.body)).await } fn convert_streaming_response(response: reqwest::Response, model: &str) -> GoogleAIProxyResponse { @@ -535,10 +522,15 @@ fn convert_streaming_response(response: reqwest::Response, model: &str) -> Googl let openai_sse_stream = async_stream::stream! { tokio::pin!(gemini_sse_stream); let mut tool_call_index: usize = 0; - while let Some(event) = gemini_sse_stream.next().await { + let mut turn = GeminiTurn::default(); + let failure = loop { + let Some(event) = gemini_sse_stream.next().await else { + break (!turn.complete_at_close()).then(truncated_stream_error); + }; match event { Ok(event) => match parse_gemini_sse_event(&event.data) { Ok(Some(parsed)) => { + turn.observe(&parsed); for chunk in gemini_event_to_openai_sse_chunks( &parsed, &id, &model, &mut tool_call_index, ) { @@ -546,12 +538,23 @@ fn convert_streaming_response(response: reqwest::Response, model: &str) -> Googl } } Ok(None) => {} - Err(e) => tracing::error!("Error parsing Gemini SSE event: {}", e), + Err(e) => break Some(e), }, + Err(EventStreamError::Transport(_)) if turn.ended() => break None, + Err(EventStreamError::Transport(e)) => { + break Some(transient_error( + format!("The connection to Gemini broke off mid-response: {e}"), + None, + )); + } Err(e) => tracing::error!("Error reading Gemini SSE stream: {}", e), } - } - yield Ok::(Bytes::from("data: [DONE]\n\n")); + }; + let last = match failure { + Some(error) => openai_stream_error_chunk(&error), + None => "data: [DONE]\n\n".to_string(), + }; + yield Ok::(Bytes::from(last)); } .boxed(); diff --git a/backend/windmill-ai/src/providers/other.rs b/backend/windmill-ai/src/providers/other.rs index 1ea54a7e79..fea8edbdac 100644 --- a/backend/windmill-ai/src/providers/other.rs +++ b/backend/windmill-ai/src/providers/other.rs @@ -266,6 +266,7 @@ impl QueryBuilder for OtherQueryBuilder { mut events_str, stream_event_processor, usage: openai_usage, + .. } = openai_sse_parser; // Process streaming events with error handling diff --git a/backend/windmill-ai/src/retry.rs b/backend/windmill-ai/src/retry.rs new file mode 100644 index 0000000000..9de9d309da --- /dev/null +++ b/backend/windmill-ai/src/retry.rs @@ -0,0 +1,474 @@ +use std::time::Duration; + +use reqwest::{header::HeaderMap, RequestBuilder, Response, StatusCode}; +use windmill_common::error::{to_anyhow, Error}; + +/// How many times a provider call is retried after its first attempt. +pub const MAX_RETRIES: u32 = 3; +const BASE_DELAY: Duration = Duration::from_secs(1); +const MAX_BACKOFF: Duration = Duration::from_secs(10); +/// A provider that asks for a longer wait is out of capacity for longer than anyone should +/// block on (a spent daily quota, say), so its error is returned instead of waited out. +const MAX_RETRY_AFTER: Duration = Duration::from_secs(60); + +/// A provider call that failed for a reason a later attempt may not meet: the provider was +/// overloaded, throttled or unreachable, or its response broke off part way. +#[derive(Debug)] +pub struct TransientAIError { + pub message: String, + /// The wait the provider asked for, when it named one. + pub retry_after: Option, +} + +impl std::fmt::Display for TransientAIError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.message) + } +} + +impl std::error::Error for TransientAIError {} + +pub fn transient_error(message: impl Into, retry_after: Option) -> Error { + to_anyhow(TransientAIError { message: message.into(), retry_after }).into() +} + +pub fn as_transient(error: &Error) -> Option<&TransientAIError> { + match error { + Error::Anyhow { error, .. } => error.downcast_ref::(), + _ => None, + } +} + +/// The error to report once a transient failure is no longer retried, without the source +/// location `Error::Anyhow` renders. +pub fn into_final_error(error: Error) -> Error { + match as_transient(&error) { + Some(transient) => Error::AIError(transient.message.clone()), + None => error, + } +} + +/// The statuses the OpenAI and Anthropic SDKs and the Vercel AI SDK retry. +fn is_retryable_status(status: u16) -> bool { + matches!(status, 408 | 409 | 429 | 500..=599) +} + +/// The error for a provider's error response. The rules follow the OpenAI and Anthropic +/// SDKs (`x-should-retry`, then the status) and gemini-cli (a daily quota is spent until +/// tomorrow, and the wait Gemini asks for is in its body). +pub fn response_error( + status: StatusCode, + headers: &HeaderMap, + body: &str, + message: String, +) -> Error { + let retryable = match headers.get("x-should-retry").and_then(|v| v.to_str().ok()) { + Some("true") => true, + Some("false") => false, + _ => { + is_retryable_status(status.as_u16()) + // OpenAI answers a spent balance with the 429 it also uses for rate limits. + && !body.contains("insufficient_quota") + && !google_quota_spent_for_the_day(body) + } + }; + if retryable { + transient_error( + message, + retry_after(headers).or_else(|| google_retry_delay(body)), + ) + } else { + Error::AIError(message) + } +} + +/// The `details` of a Google API error body, where Gemini puts its quota and retry hints. +fn google_error_details(body: &str) -> Vec { + serde_json::from_str::(body) + .ok() + .and_then(|v| v.get("error")?.get("details")?.as_array().cloned()) + .unwrap_or_default() +} + +fn google_quota_spent_for_the_day(body: &str) -> bool { + google_error_details(body).iter().any(|detail| { + detail + .get("violations") + .and_then(|v| v.as_array()) + .is_some_and(|violations| { + violations.iter().any(|violation| { + violation + .get("quotaId") + .and_then(|id| id.as_str()) + .is_some_and(|id| id.contains("PerDay") || id.contains("Daily")) + }) + }) + }) +} + +/// The `RetryInfo.retryDelay` of a Google API error, a protobuf duration such as `"53s"`. +fn google_retry_delay(body: &str) -> Option { + google_error_details(body).iter().find_map(|detail| { + let delay = detail.get("retryDelay")?.as_str()?.strip_suffix('s')?; + let seconds = delay + .parse::() + .ok() + .filter(|v| v.is_finite() && *v >= 0.0)?; + Some(Duration::try_from_secs_f64(seconds).unwrap_or(Duration::MAX)) + }) +} + +/// The wait a provider asked for in its `retry-after-ms` or `retry-after` header. An +/// HTTP-date `retry-after` is ignored, which falls back to the computed backoff. +fn retry_after(headers: &HeaderMap) -> Option { + let seconds = |name: &str, scale: f64| { + headers + .get(name)? + .to_str() + .ok()? + .trim() + .parse::() + .ok() + .filter(|v| v.is_finite() && *v >= 0.0) + .map(|v| Duration::try_from_secs_f64(v / scale).unwrap_or(Duration::MAX)) + }; + seconds("retry-after-ms", 1000.0).or_else(|| seconds("retry-after", 1.0)) +} + +/// The error object a provider sends inside a stream: OpenAI-compatible chunks carry +/// `type` and a string or status `code`, Anthropic `type`, Gemini a status `code` and +/// `status`. +#[derive(Debug, Default, serde::Deserialize)] +pub struct StreamErrorBody { + #[serde(default)] + pub message: Option, + #[serde(default, rename = "type")] + pub kind: Option, + #[serde(default)] + pub code: Option, + #[serde(default)] + pub status: Option, +} + +impl StreamErrorBody { + /// The error for a failure an OpenAI-compatible or Gemini stream reported after opening + /// with a 200, classified as the Vercel AI SDK classifies OpenAI stream errors + /// (`openai-stream-error.ts`): the status `code` holds, else the status its code and type + /// name, else a server error. + pub fn into_error(self, provider: &str) -> Error { + let code = self.code_str(); + let discriminator = [code.as_deref(), self.kind.as_deref()] + .into_iter() + .flatten() + .collect::>() + .join(" ") + .to_lowercase(); + let has = |terms: &[&str]| terms.iter().any(|term| discriminator.contains(term)); + let status = match code.as_deref().and_then(|code| code.parse::().ok()) { + Some(status @ 400..=599) => status, + _ if has(&["insufficient_quota", "rate_limit"]) => 429, + _ if has(&["authentication"]) => 401, + _ if has(&["permission"]) => 403, + _ if has(&["not_found"]) => 404, + _ if has(&["invalid", "bad_request", "context_length"]) => 400, + _ if has(&["overload"]) => 503, + _ if has(&["timeout"]) => 504, + _ => 500, + }; + let retryable = !has(&["insufficient_quota"]) && is_retryable_status(status); + self.into_classified_error(provider, retryable) + } + + /// The error for a failure an Anthropic stream reported after opening with a 200. The + /// kinds retried are those the Vercel AI SDK retries, plus `timeout_error`, which + /// Anthropic documents as its 504; any other kind is a fault in the request or account. + pub fn into_anthropic_error(self) -> Error { + let retryable = matches!( + self.kind.as_deref(), + Some("api_error" | "overloaded_error" | "rate_limit_error" | "timeout_error") + ); + self.into_classified_error("Anthropic", retryable) + } + + fn code_str(&self) -> Option { + match self.code.as_ref()? { + serde_json::Value::String(code) => Some(code.clone()), + serde_json::Value::Number(code) => Some(code.to_string()), + _ => None, + } + } + + fn into_classified_error(self, provider: &str, retryable: bool) -> Error { + let kind = [self.kind.clone(), self.code_str(), self.status] + .into_iter() + .flatten() + .collect::>() + .join("/"); + let message = format!( + "{provider} stream error{}: {}", + (!kind.is_empty()) + .then(|| format!(" ({kind})")) + .unwrap_or_default(), + self.message.as_deref().unwrap_or("no details") + ); + if retryable { + transient_error(message, None) + } else { + Error::AIError(message) + } + } +} + +/// The error for a stream that closed before the provider sent the event ending a +/// complete response: cut off by the provider or by a proxy in between. +pub fn truncated_stream_error() -> Error { + transient_error( + "The AI provider's response stream ended before the response was complete", + None, + ) +} + +/// An OpenAI-format stream chunk reporting `error`, which ends a response converted to that +/// format after it started streaming: the chat's OpenAI SDK throws on a chunk carrying +/// `error`, where a bare end of stream would read as a finished answer. +pub fn openai_stream_error_chunk(error: &Error) -> String { + let message = as_transient(error).map_or_else(|| error.to_string(), |t| t.message.clone()); + format!( + "data: {}\n\n", + serde_json::json!({ "error": { "message": message } }) + ) +} + +/// Sends a provider request, retrying a transient failure while nothing of the response +/// has been read: a failed connection or an error status. Once retries are spent the error +/// carries the provider's answer, as a single attempt's would. +pub async fn send_with_retries(request: RequestBuilder) -> Result { + let mut backoff = Backoff::default(); + loop { + // A streamed body cannot be replayed, so such a request is sent once. + let Some(attempt) = request.try_clone() else { + return send_once(request).await.map_err(into_final_error); + }; + let error = match send_once(attempt).await { + Ok(response) => return Ok(response), + Err(error) => error, + }; + let Some(delay) = as_transient(&error).and_then(|t| backoff.next_delay(t)) else { + return Err(into_final_error(error)); + }; + tracing::warn!( + "AI provider call failed ({error}); retrying in {:.1}s ({}/{MAX_RETRIES})", + delay.as_secs_f64(), + backoff.retries() + ); + tokio::time::sleep(delay).await; + } +} + +/// The error for a provider request that failed before any response arrived. The request +/// timeout is the caller's whole budget, so a timed-out call leaves no time for another +/// attempt; a connect timeout has its own, shorter one. +pub fn send_error(e: reqwest::Error, context: &str) -> Error { + let message = format!("{context}: {e}"); + if e.is_builder() || (e.is_timeout() && !e.is_connect()) { + Error::AIError(message) + } else { + transient_error(message, None) + } +} + +async fn send_once(request: RequestBuilder) -> Result { + let response = request + .send() + .await + .map_err(|e| send_error(e, "Failed to reach the AI provider"))?; + let status = response.status(); + if status.is_client_error() || status.is_server_error() { + let headers = response.headers().clone(); + let body = response.text().await.unwrap_or_default(); + return Err(response_error(status, &headers, &body, body.clone())); + } + Ok(response) +} + +/// Counts the retries of one provider call and spaces them out. +#[derive(Debug, Default)] +pub struct Backoff { + retries: u32, +} + +impl Backoff { + /// The wait before retrying a call that failed with `error`, or `None` once it should + /// fail instead. + pub fn next_delay(&mut self, error: &TransientAIError) -> Option { + if self.retries >= MAX_RETRIES { + return None; + } + let delay = match error.retry_after { + Some(wait) if wait > MAX_RETRY_AFTER => return None, + // A zero wait would retry back to back, so it gets the computed backoff. + Some(wait) if !wait.is_zero() => wait, + // Jittered, so callers throttled together do not come back together. + _ => (BASE_DELAY * 2u32.pow(self.retries)) + .min(MAX_BACKOFF) + .mul_f64(0.5 + rand::random::() * 0.5), + }; + self.retries += 1; + Some(delay) + } + + /// Retries granted so far. + pub fn retries(&self) -> u32 { + self.retries + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Error responses as providers send them, each with the retry decision of the client + /// the rule is taken from. + #[test] + fn classifies_error_responses() { + const GEMINI_QUOTA: &str = r#"{"error":{"code":429,"status":"RESOURCE_EXHAUSTED","details":[{"@type":"type.googleapis.com/google.rpc.QuotaFailure","violations":[{"quotaId":"QUOTA_ID"}]},{"@type":"type.googleapis.com/google.rpc.RetryInfo","retryDelay":"53s"}]}}"#; + let cases: &[(u16, &[(&str, &str)], &str, Option>)] = &[ + ( + 429, + &[], + r#"{"error":{"code":"rate_limit_exceeded"}}"#, + Some(None), + ), + ( + 429, + &[], + r#"{"error":{"type":"insufficient_quota","code":"insufficient_quota"}}"#, + None, + ), + ( + 529, + &[], + r#"{"type":"error","error":{"type":"overloaded_error"}}"#, + Some(None), + ), + (501, &[], "", Some(None)), + (400, &[], "", None), + (401, &[], "", None), + (400, &[("x-should-retry", "true")], "", Some(None)), + (503, &[("x-should-retry", "false")], "", None), + (429, &[("retry-after", "2")], "", Some(Some(2))), + ( + 429, + &[], + &GEMINI_QUOTA.replace("QUOTA_ID", "GenerateRequestsPerMinutePerProjectPerModel"), + Some(Some(53)), + ), + ( + 429, + &[], + &GEMINI_QUOTA.replace( + "QUOTA_ID", + "GenerateRequestsPerDayPerProjectPerModel-FreeTier", + ), + None, + ), + ]; + for (status, headers, body, expected) in cases { + let headers = headers + .iter() + .map(|(k, v)| (k.parse().unwrap(), v.parse().unwrap())) + .collect::(); + let error = response_error( + StatusCode::from_u16(*status).unwrap(), + &headers, + body, + String::new(), + ); + let got = as_transient(&error).map(|t| t.retry_after.map(|d| d.as_secs())); + assert_eq!(&got, expected, "{status} {headers:?} {body}"); + } + } + + /// In-stream error objects as providers send them, after the 200 opened the stream. + #[test] + fn classifies_stream_errors() { + let retried = |json: &str, anthropic: bool| { + let body: StreamErrorBody = serde_json::from_str(json).unwrap(); + let error = if anthropic { + body.into_anthropic_error() + } else { + body.into_error("AI provider") + }; + as_transient(&error).is_some() + }; + for (json, expected) in [ + (r#"{"message":"m","type":"server_error"}"#, true), + ( + r#"{"message":"m","type":"requests","code":"rate_limit_exceeded"}"#, + true, + ), + ( + r#"{"message":"m","type":"insufficient_quota","code":"insufficient_quota"}"#, + false, + ), + ( + r#"{"message":"m","type":"invalid_request_error","code":"context_length_exceeded"}"#, + false, + ), + // LiteLLM's proxy puts the upstream status in `code` as a string. + (r#"{"message":"m","type":"None","code":"400"}"#, false), + (r#"{"message":"m","code":"503"}"#, true), + (r#"{"code":503,"message":"m","status":"UNAVAILABLE"}"#, true), + ( + r#"{"code":400,"message":"m","status":"INVALID_ARGUMENT"}"#, + false, + ), + (r#"{"message":"m"}"#, true), + ] { + assert_eq!(retried(json, false), expected, "{json}"); + } + for (kind, expected) in [ + ("overloaded_error", true), + ("api_error", true), + ("rate_limit_error", true), + ("timeout_error", true), + ("billing_error", false), + ("invalid_request_error", false), + ("some_new_error", false), + ] { + let json = format!(r#"{{"type":"{kind}","message":"m"}}"#); + assert_eq!(retried(&json, true), expected, "{kind}"); + } + } + + #[test] + fn honours_retry_after_up_to_a_bound() { + let mut headers = HeaderMap::new(); + headers.insert("retry-after", "2".parse().unwrap()); + assert_eq!(retry_after(&headers), Some(Duration::from_secs(2))); + headers.insert("retry-after-ms", "1500".parse().unwrap()); + assert_eq!(retry_after(&headers), Some(Duration::from_millis(1500))); + headers.remove("retry-after-ms"); + headers.insert("retry-after", "1e30".parse().unwrap()); + assert_eq!(retry_after(&headers), Some(Duration::MAX)); + + let mut backoff = Backoff::default(); + let asked = |secs| TransientAIError { + message: String::new(), + retry_after: Some(Duration::from_secs(secs)), + }; + assert_eq!(backoff.next_delay(&asked(5)), Some(Duration::from_secs(5))); + assert_eq!(backoff.next_delay(&asked(3600)), None); + } + + #[test] + fn stops_after_max_retries() { + let mut backoff = Backoff::default(); + let error = TransientAIError { message: String::new(), retry_after: None }; + for _ in 0..MAX_RETRIES { + let delay = backoff.next_delay(&error).expect("retry"); + assert!(delay <= MAX_BACKOFF); + } + assert_eq!(backoff.next_delay(&error), None); + } +} diff --git a/backend/windmill-ai/src/sse.rs b/backend/windmill-ai/src/sse.rs index ada50afca5..bdaa75025f 100644 --- a/backend/windmill-ai/src/sse.rs +++ b/backend/windmill-ai/src/sse.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; -use eventsource_stream::Eventsource; +use eventsource_stream::{EventStreamError, Eventsource}; use indexmap::IndexMap; use reqwest::Response; use serde::Deserialize; @@ -8,12 +8,13 @@ use tokio_stream::StreamExt; use windmill_common::{error::Error, utils::rd_string}; use crate::{ - ai_google::{parse_gemini_sse_event, GeminiUsageMetadata}, + ai_google::{parse_gemini_sse_event, GeminiTurn, GeminiUsageMetadata}, ai_types::UrlCitation, ai_types::{ AnthropicExtraContent, ExtraContent, GoogleExtraContent, OpenAIFunction, OpenAIToolCall, }, query_builder::StreamEventSink, + retry::{transient_error, truncated_stream_error, StreamErrorBody}, types::{StreamingEvent, TokenUsage}, }; @@ -43,6 +44,8 @@ pub struct OpenAIChoiceDelta { #[derive(Deserialize)] pub struct OpenAIChoice { pub delta: Option, + #[serde(default)] + pub finish_reason: Option, } /// Nested prompt token details returned by the Chat Completions API. @@ -87,6 +90,8 @@ pub struct OpenAISSEEvent { pub choices: Option>, #[serde(default)] pub usage: Option, + #[serde(default)] + pub error: Option, } lazy_static::lazy_static! { @@ -99,6 +104,14 @@ lazy_static::lazy_static! { pub trait SSEParser { async fn parse_event_data(&mut self, data: &str) -> Result<(), Error>; + /// Whether the stream delivered the event that ends a complete response. + fn is_complete(&self) -> bool; + + /// Whether a stream that closed cleanly holds a complete response. + fn is_complete_at_close(&self) -> bool { + self.is_complete() + } + async fn parse_events(&mut self, response: Response) -> Result<(), Error> { let mut stream = response.bytes_stream().eventsource(); let mut consecutive_errors = 0; @@ -113,6 +126,18 @@ pub trait SSEParser { self.parse_event_data(&event.data).await?; } + // The body ends right after a transport error, so it cannot be skipped + // like a malformed event. A timeout spent the caller's whole budget. + Err(EventStreamError::Transport(_)) if self.is_complete() => break, + Err(EventStreamError::Transport(e)) => { + let message = + format!("The connection to the AI provider broke off mid-response: {e}"); + return Err(if e.is_timeout() { + Error::AIError(message) + } else { + transient_error(message, None) + }); + } Err(e) => { consecutive_errors += 1; tracing::error!("Failed to parse SSE event: {}", e); @@ -130,6 +155,9 @@ pub trait SSEParser { } } + if !self.is_complete_at_close() { + return Err(truncated_stream_error()); + } Ok(()) } } @@ -145,6 +173,9 @@ pub struct OpenAISSEParser { pub stream_event_processor: Box, /// Token usage from final chunk (when stream_options.include_usage is true) pub usage: Option, + /// Set by `[DONE]` or by a choice's `finish_reason`: OpenAI-compatible gateways do not + /// all send `[DONE]`. + complete: bool, } impl OpenAISSEParser { @@ -156,13 +187,19 @@ impl OpenAISSEParser { events_str: String::new(), stream_event_processor, usage: None, + complete: false, } } } impl SSEParser for OpenAISSEParser { + fn is_complete(&self) -> bool { + self.complete + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { if data == "[DONE]" { + self.complete = true; return Ok(()); } @@ -173,13 +210,21 @@ impl SSEParser for OpenAISSEParser { .ok(); if let Some(event) = event { + if let Some(error) = event.error { + return Err(error.into_error("AI provider")); + } + // Extract usage from final chunk (when stream_options.include_usage is true) if let Some(usage) = event.usage { self.usage = Some(usage); } if let Some(mut choices) = event.choices.filter(|s| !s.is_empty()) { - if let Some(delta) = choices.remove(0).delta { + let choice = choices.remove(0); + if choice.finish_reason.is_some() { + self.complete = true; + } + if let Some(delta) = choice.delta { if let Some(reasoning) = delta.reasoning_content.filter(|s| !s.is_empty()) { self.accumulated_reasoning.push_str(&reasoning); let event = StreamingEvent::ReasoningTokenDelta { content: reasoning }; @@ -345,6 +390,8 @@ pub enum AnthropicSSEEvent { MessageDelta { #[serde(default)] usage: Option, + #[serde(default)] + delta: Option, }, #[serde(rename = "message_stop")] MessageStop {}, @@ -353,12 +400,18 @@ pub enum AnthropicSSEEvent { #[serde(rename = "error")] Error { #[serde(default)] - message: Option, + error: StreamErrorBody, }, #[serde(other)] Unknown, } +#[derive(Deserialize, Debug)] +pub struct AnthropicMessageDelta { + #[serde(default)] + pub stop_reason: Option, +} + /// Tracks state of a content block during streaming #[derive(Debug)] #[allow(dead_code)] @@ -391,6 +444,9 @@ pub struct AnthropicSSEParser { /// can be replayed before `tool_use` (required by Claude when thinking is on). pending_reasoning: Option, reasoning_attached: bool, + /// Set by `message_stop` or by the `stop_reason` of the `message_delta` sent just + /// before it, so a gateway that drops `message_stop` does not fail a whole answer. + complete: bool, } impl AnthropicSSEParser { @@ -407,11 +463,16 @@ impl AnthropicSSEParser { usage: None, pending_reasoning: None, reasoning_attached: false, + complete: false, } } } impl SSEParser for AnthropicSSEParser { + fn is_complete(&self) -> bool { + self.complete + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { let event: Option = serde_json::from_str(data) .inspect_err(|e| { @@ -583,9 +644,8 @@ impl SSEParser for AnthropicSSEParser { .await?; } } - AnthropicSSEEvent::Error { message } => { - let error_msg = message.unwrap_or_else(|| "Unknown error".to_string()); - tracing::error!("Anthropic streaming error: {}", error_msg); + AnthropicSSEEvent::Error { error } => { + return Err(error.into_anthropic_error()); } AnthropicSSEEvent::MessageStart { message } => { // The only event carrying the prompt-side counts. `message_delta` @@ -595,7 +655,10 @@ impl SSEParser for AnthropicSSEParser { self.usage = Some(usage); } } - AnthropicSSEEvent::MessageDelta { usage } => { + AnthropicSSEEvent::MessageDelta { usage, delta } => { + if delta.is_some_and(|d| d.stop_reason.is_some()) { + self.complete = true; + } if let Some(u) = usage { match &mut self.usage { // Field by field, so the prompt counts from `message_start` @@ -614,10 +677,9 @@ impl SSEParser for AnthropicSSEParser { } } } + AnthropicSSEEvent::MessageStop {} => self.complete = true, // Ignore other events - AnthropicSSEEvent::MessageStop {} - | AnthropicSSEEvent::Ping {} - | AnthropicSSEEvent::Unknown => {} + AnthropicSSEEvent::Ping {} | AnthropicSSEEvent::Unknown => {} } } @@ -645,6 +707,7 @@ pub struct GeminiSSEParser { pub annotations: Vec, pub used_websearch: bool, pub usage: Option, + turn: GeminiTurn, } impl GeminiSSEParser { @@ -659,15 +722,25 @@ impl GeminiSSEParser { annotations: Vec::new(), used_websearch: false, usage: None, + turn: GeminiTurn::default(), } } } impl SSEParser for GeminiSSEParser { + fn is_complete(&self) -> bool { + self.turn.ended() + } + + fn is_complete_at_close(&self) -> bool { + self.turn.complete_at_close() + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { let Some(parsed) = parse_gemini_sse_event(data)? else { return Ok(()); }; + self.turn.observe(&parsed); if let Some(reasoning) = parsed.reasoning.filter(|s| !s.is_empty()) { self.accumulated_reasoning.push_str(&reasoning); @@ -797,6 +870,8 @@ impl OpenAIResponsesUsage { pub struct OpenAIResponsesResponse { #[serde(default)] pub usage: Option, + #[serde(default)] + pub error: Option, } /// SSE event types for OpenAI Responses API streaming @@ -833,6 +908,20 @@ pub enum OpenAIResponsesSSEEvent { #[serde(rename = "response.incomplete")] Incomplete { response: OpenAIResponsesResponse }, + #[serde(rename = "response.failed")] + Failed { response: OpenAIResponsesResponse }, + + /// Flat (`code`, `message`) from OpenAI, nested under `error` from some compatible servers. + #[serde(rename = "error")] + Error { + #[serde(default)] + code: Option, + #[serde(default)] + message: Option, + #[serde(default)] + error: Option, + }, + /// Response created #[serde(rename = "response.created")] Created {}, @@ -890,6 +979,7 @@ pub struct OpenAIResponsesSSEParser { pub usage: Option, /// Reasoning summary parts seen so far, to separate them as paragraphs reasoning_summary_parts: usize, + response_ended: bool, } impl OpenAIResponsesSSEParser { @@ -906,11 +996,16 @@ impl OpenAIResponsesSSEParser { used_websearch: false, usage: None, reasoning_summary_parts: 0, + response_ended: false, } } } impl SSEParser for OpenAIResponsesSSEParser { + fn is_complete(&self) -> bool { + self.response_ended + } + async fn parse_event_data(&mut self, data: &str) -> Result<(), Error> { let event: Option = serde_json::from_str(data) .inspect_err(|e| { @@ -1010,11 +1105,22 @@ impl SSEParser for OpenAIResponsesSSEParser { OpenAIResponsesSSEEvent::Completed { response } | OpenAIResponsesSSEEvent::Incomplete { response } => { + self.response_ended = true; if let Some(usage) = response.usage { self.usage = Some(usage); } } + OpenAIResponsesSSEEvent::Failed { response } => { + return Err(response.error.unwrap_or_default().into_error("OpenAI")); + } + + OpenAIResponsesSSEEvent::Error { code, message, error } => { + return Err(error + .unwrap_or(StreamErrorBody { code, message, ..Default::default() }) + .into_error("OpenAI")); + } + OpenAIResponsesSSEEvent::ReasoningSummaryPartAdded {} => { self.reasoning_summary_parts += 1; if self.reasoning_summary_parts > 1 { @@ -1037,9 +1143,10 @@ impl SSEParser for OpenAIResponsesSSEParser { } } + OpenAIResponsesSSEEvent::Done {} => self.response_ended = true, + // Ignore other event types - OpenAIResponsesSSEEvent::Done {} - | OpenAIResponsesSSEEvent::Created {} + OpenAIResponsesSSEEvent::Created {} | OpenAIResponsesSSEEvent::InProgress {} | OpenAIResponsesSSEEvent::OutputItemDone {} | OpenAIResponsesSSEEvent::ContentPartAdded {} @@ -1093,6 +1200,117 @@ mod tests { } } + fn sse_response(chunks: Vec>) -> Response { + let body = reqwest::Body::wrap_stream(futures::stream::iter(chunks)); + Response::from(http::Response::new(body)) + } + + /// A stream that breaks off, or reports a failure inside its 200, must fail the call + /// rather than hand back what arrived as if it were the whole answer. + #[tokio::test] + async fn stream_failures_are_errors_not_partial_answers() { + use crate::retry::as_transient; + + const STARTED: &str = concat!( + "data: {\"type\":\"message_start\"}\n\n", + "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"text\",\"text\":\"\"}}\n\n", + "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"text_delta\",\"text\":\"Hel\"}}\n\n", + ); + let anthropic = |chunks| async move { + AnthropicSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(chunks)) + .await + }; + + let overloaded = anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"error\",\"error\":{\"type\":\"overloaded_error\",\"message\":\"Overloaded\"}}\n\n"), + ]) + .await + .unwrap_err(); + assert!(as_transient(&overloaded).is_some(), "{overloaded}"); + + let refused = anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"bad\"}}\n\n"), + ]) + .await + .unwrap_err(); + assert!(as_transient(&refused).is_none(), "{refused}"); + + let truncated = anthropic(vec![Ok(STARTED)]).await.unwrap_err(); + assert!(as_transient(&truncated).is_some(), "{truncated}"); + + let dropped = anthropic(vec![ + Ok(STARTED), + Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "reset", + )), + ]) + .await + .unwrap_err(); + assert!(as_transient(&dropped).is_some(), "{dropped}"); + + anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"message_stop\"}\n\n"), + ]) + .await + .unwrap(); + + anthropic(vec![ + Ok(STARTED), + Ok("data: {\"type\":\"message_delta\",\"delta\":{\"stop_reason\":\"end_turn\"}}\n\n"), + ]) + .await + .unwrap(); + + let truncated = OpenAISSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(vec![Ok( + "data: {\"choices\":[{\"delta\":{\"content\":\"Hi\"}}]}\n\n", + )])) + .await + .unwrap_err(); + assert!(as_transient(&truncated).is_some(), "{truncated}"); + + let quota = OpenAIResponsesSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(vec![Ok( + "data: {\"type\":\"error\",\"error\":{\"type\":\"insufficient_quota\",\"code\":\"insufficient_quota\",\"message\":\"m\"}}\n\n", + )])) + .await + .unwrap_err(); + assert!(as_transient("a).is_none(), "{quota}"); + + GeminiSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(vec![Ok( + "data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"functionCall\":{\"name\":\"get_weather\",\"args\":{}}}]}}]}\n\n", + )])) + .await + .unwrap(); + + let gemini = |chunks| async move { + GeminiSSEParser::new(Box::new(NoopSink)) + .parse_events(sse_response(chunks)) + .await + }; + // Thinking spent the whole `maxOutputTokens`: a clean close with no event. + gemini(vec![]).await.unwrap(); + let dropped = gemini(vec![Err(std::io::Error::new( + std::io::ErrorKind::ConnectionReset, + "reset", + ))]) + .await + .unwrap_err(); + assert!(as_transient(&dropped).is_some(), "{dropped}"); + let truncated = gemini(vec![Ok( + "data: {\"candidates\":[{\"content\":{\"role\":\"model\",\"parts\":[{\"text\":\"Hi\"}]}}]}\n\n", + )]) + .await + .unwrap_err(); + assert!(as_transient(&truncated).is_some(), "{truncated}"); + } + /// The prompt-side counts arrive only on `message_start` and the completion total /// only on `message_delta`; a parser that keeps just the last one reports a request /// with no input tokens at all. diff --git a/backend/windmill-api-auth/src/scopes.rs b/backend/windmill-api-auth/src/scopes.rs index b636e11a1c..21f916bf94 100644 --- a/backend/windmill-api-auth/src/scopes.rs +++ b/backend/windmill-api-auth/src/scopes.rs @@ -170,10 +170,16 @@ const FLOW_JOBS: [&'static str; 6] = [ /// scoped by the agent's path like a deployed runnable's, as `jobs:run:agents:`. const AGENT_JOBS: [&'static str; 1] = ["jobs/run/agent"]; +/// Lock resolution for scripts, apps and flows, scoped as `jobs:run:dependencies:`. +/// Resolving a package can run its build or install scripts on the worker, so this is +/// still code execution: the scope only takes away the direct ways to run code. +const DEPENDENCY_JOBS: [&'static str; 2] = ["jobs/run/dependencies", "jobs/run/flow_dependencies"]; + lazy_static::lazy_static! { static ref RUN_PATH_ACTIONS: Vec<&'static str> = { - let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "jobs/run/dependencies","jobs/run/flow_dependencies", "apps_u/execute_component", "apps_u/upload_s3_file"]; + let mut v = vec!["jobs/resume/", "jobs/run/batch_rerun_jobs", "jobs/run/workflow_as_code", "apps_u/execute_component", "apps_u/upload_s3_file"]; + v.extend(DEPENDENCY_JOBS); v.extend(SCRIPT_JOBS); v.extend(FLOW_JOBS); v.extend(AGENT_JOBS); @@ -227,11 +233,13 @@ fn map_http_method_to_action(method: &str, route_path: &str) -> ScopeAction { } } -/// Checks the route path to determine the runnable kind ("agents", "flows" or "scripts"). +/// Checks the route path to determine the runnable kind ("dependencies", "agents", "flows" +/// or "scripts"). /// /// The order of checks is important: -/// - Flow-related paths are checked first to avoid false positives, as some flow paths -/// (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`). +/// - Dependency paths are checked before flow paths: `flow_dependencies` starts with `f`. +/// - Flow-related paths are checked before script paths to avoid false positives, as some +/// flow paths (e.g., `/run_preview_flow`) share prefixes with script paths (e.g., `/run_preview`). /// /// Returns the kind based on the match, or `None` if no match is found. fn determine_kind_from_route(route_path: &str) -> Option { @@ -247,6 +255,12 @@ fn determine_kind_from_route(route_path: &str) -> Option { { return None; } + if DEPENDENCY_JOBS + .iter() + .any(|path| route_path.starts_with(path)) + { + return Some("dependencies".to_string()); + } if AGENT_JOBS.iter().any(|path| route_path.starts_with(path)) { return Some("agents".to_string()); } @@ -586,17 +600,23 @@ pub struct JobReadConfinement { } impl JobReadConfinement { - /// Whether a job that ran `runnable_path` as `kind` (`scripts`, `flows` or `agents`) - /// is inside the confinement. A read path admits every kind; a run scope only its own. + /// Whether a job that ran `runnable_path` as `kind` (`scripts`, `flows`, `agents` or + /// `dependencies`) is inside the confinement. A read path admits every kind; a run + /// scope only its own. pub fn admits(&self, kind: &str, runnable_path: &str) -> bool { if self.read_paths.allows(runnable_path) { return true; } + // Lock jobs and previews store the path their caller sent. The matcher reads `*` + // in it as a pattern, and `,`/`:` are scope syntax: such a path needs a grant + // covering every path, as enqueuing it did. + let resource = windmill_common::auth::is_scope_literal_path(runnable_path) + .then(|| vec![runnable_path.to_string()]); let required = ScopeDefinition::new( ScopeDomain::Jobs.as_str(), ScopeAction::Run.as_str(), Some(kind), - Some(vec![runnable_path.to_string()]), + resource, ); self.run.iter().any(|scope| scope.includes(&required)) } @@ -862,7 +882,8 @@ fn scope_grants_access( /// from its own super-admin gate, these among them, so each is already readable by every /// authenticated user; each is here because the CLI reads it from a job: /// `automate_username_creation` before creating a user during a git-sync push, `uid` and -/// `hub_base_url` when `u/admin/hub_sync` pulls resource types from the Hub. The other +/// `hub_base_url` when a job runs `wmill hub pull`, as an edited copy of the retired +/// `u/admin/hub_sync` still does (servers sync resource types themselves). The other /// ungated keys have no such caller, so they stay confined — being ungated earns a key /// nothing on its own. Listing a gated key earns it nothing either: `require_super_admin` /// refuses every job token, so `hub_api_secret`, which that pull reads for a private Hub, @@ -1714,6 +1735,24 @@ mod tests { scope_for_route("POST", "/api/w/ws/jobs/run/agent/u/x/y").as_deref(), Some("jobs:run:agents") ); + for path in [ + "/api/w/ws/jobs/run/dependencies", + "/api/w/ws/jobs/run/dependencies_async", + "/api/w/ws/jobs/run/flow_dependencies", + "/api/w/ws/jobs/run/flow_dependencies_async", + ] { + assert_eq!( + scope_for_route("POST", path).as_deref(), + Some("jobs:run:dependencies"), + "{path}" + ); + assert!(check_route_access(&["jobs:run:dependencies".into()], path, "POST").is_ok()); + assert!(check_route_access(&["jobs:run:flows".into()], path, "POST").is_err()); + } + // A dependencies grant reaches no other run route. + for path in ["/api/w/ws/jobs/run/preview", "/api/w/ws/jobs/run/p/u/x/y"] { + assert!(check_route_access(&["jobs:run:dependencies".into()], path, "POST").is_err()); + } // Preview/bundle runs have no deployed path and their handlers require the // broad `jobs:run` scope, so the derived scope must not carry a kind. diff --git a/backend/windmill-api-flows/src/flows.rs b/backend/windmill-api-flows/src/flows.rs index 1007cfd5b1..eab73bc015 100644 --- a/backend/windmill-api-flows/src/flows.rs +++ b/backend/windmill-api-flows/src/flows.rs @@ -20,7 +20,8 @@ use windmill_api_auth::{ require_owner_of_path, ApiAuthed, }; use windmill_common::workspaces::{ - check_deploy_rules, check_operator_can_build_flows, operator_can_build_flows, RuleCheckResult, + check_deploy_rules, check_operator_can_build, operator_builder_rights, BuilderKind, + RuleCheckResult, }; use windmill_common::{ user_drafts::{overlay_or_draft_only, DraftUserRef, UserDraftItemKind, WithDraftOverlay}, @@ -243,7 +244,7 @@ async fn list_flows( // Append the authed user's drafts at paths with no deployed flow; see scripts.rs. if lq.include_draft_only.unwrap_or(false) - && (!authed.is_operator || operator_can_build_flows(&db, &w_id).await?) + && (!authed.is_operator || operator_builder_rights(&db, &w_id).await?.flows) && offset == 0 && lq.path_start.is_none() && lq.path_exact.is_none() @@ -731,7 +732,14 @@ async fn create_flow( Path(w_id): Path, Json(mut nf): Json, ) -> Result<(StatusCode, String)> { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "create flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "create flows", + ) + .await?; check_scopes(&authed, || format!("flows:write:{}", nf.path))?; // A `<= 0` flow timeout is "unset", not a 0-second limit that kills every run instantly. @@ -1344,7 +1352,14 @@ async fn update_flow( Path((w_id, flow_path)): Path<(String, StripPath)>, Json(ef): Json, ) -> Result { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "update flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "update flows", + ) + .await?; let flow_path = flow_path.to_path(); // The URL identifies the flow being updated; the body path is only needed to rename. let mut nf = ef.into_new_flow(flow_path); @@ -2038,7 +2053,14 @@ async fn archive_flow_by_path( Path((w_id, path)): Path<(String, StripPath)>, Json(archived): Json, ) -> Result { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "archive flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "archive flows", + ) + .await?; let path = path.to_path(); check_scopes(&authed, || format!("flows:write:{}", path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( @@ -2179,7 +2201,14 @@ async fn delete_flow_by_path( Path((w_id, path)): Path<(String, StripPath)>, Query(query): Query, ) -> Result { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "delete flows").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "delete flows", + ) + .await?; let path = path.to_path(); check_scopes(&authed, || format!("flows:write:{}", path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( diff --git a/backend/windmill-api-integration-tests/tests/operator_builder_apps.rs b/backend/windmill-api-integration-tests/tests/operator_builder_apps.rs new file mode 100644 index 0000000000..7033d093f2 --- /dev/null +++ b/backend/windmill-api-integration-tests/tests/operator_builder_apps.rs @@ -0,0 +1,570 @@ +use serde_json::json; +use serial_test::serial; +use sqlx::{Pool, Postgres}; +use windmill_common::workspaces::invalidate_operator_rights_cache; +use windmill_test_utils::*; + +/// Every test here must be `#[serial]`: the rights cache is process-global and keyed by workspace id +/// alone, so parallel tests under this same id read each other's rights. +const WS: &str = "test-workspace"; + +fn client(token: &str) -> reqwest::Client { + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_str(&format!("Bearer {token}")).unwrap(), + ); + reqwest::ClientBuilder::new() + .default_headers(headers) + .build() + .unwrap() +} + +async fn set_builder(db: &Pool, flows: bool, apps: bool) -> anyhow::Result<()> { + sqlx::query( + "UPDATE workspace_settings SET operator_settings = $1::text::jsonb WHERE workspace_id = $2", + ) + .bind(format!( + r#"{{"builder_flows": {flows}, "builder_apps": {apps}}}"# + )) + .bind(WS) + .execute(db) + .await?; + // The rights are read through a process-global 60s cache keyed by workspace id. + invalidate_operator_rights_cache(WS); + Ok(()) +} + +async fn add_script(db: &Pool, hash: i64, path: &str, owner: &str) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, language, kind, created_by, schema, + summary, description, lock, extra_perms) + VALUES ($1, $2, $3, 'x', 'bun', 'script', $4, '{}', '', '', '', '{}')", + ) + .bind(WS) + .bind(hash) + .bind(path) + .bind(owner) + .execute(db) + .await?; + Ok(()) +} + +/// A full-code app is deployed multipart, the way the editor and the CLI use it. +fn raw_app_form(app: serde_json::Value) -> reqwest::multipart::Form { + reqwest::multipart::Form::new() + .part( + "app", + reqwest::multipart::Part::text(app.to_string()) + .mime_str("application/json") + .unwrap(), + ) + .part( + "js", + reqwest::multipart::Part::text("console.log(1)").file_name("app.js"), + ) +} + +fn raw_app(path: &str, mode: &str, runnable_path: &str) -> reqwest::multipart::Form { + raw_app_form(json!({ + "path": path, + "summary": "", + "value": {"files": {}, "runnables": {"r": { + "name": "r", "type": "runnableByPath", "runType": "script", + "path": runnable_path + }}}, + "policy": {"execution_mode": mode, "triggerables_v2": {}} + })) +} + +fn composition_flow(path: &str) -> serde_json::Value { + json!({ + "path": path, "summary": "", "description": "", "schema": {}, + "value": {"modules": [{ + "id": "a", + "value": {"type": "script", "path": "u/operator/some_script", "input_transforms": {}} + }]} + }) +} + +/// The app half of the builder boundary, plus the proof that the two rights are independent: a +/// gate reading "either" would authorize the kind its workspace never granted, which is the whole +/// point of splitting them. +#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))] +#[serial] +async fn test_operator_builder_apps_boundary(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + let api = format!("http://localhost:{port}/api/w/{WS}"); + let c = client("OPERATOR_TOKEN_1"); + + add_script(&db, 4241, "u/operator/some_script", "operator").await?; + // `permissions_test` gives the operator fixture no rights on `u/alice/**`. + add_script(&db, 4243, "u/alice/private", "alice").await?; + // A version shared with the operator, then archived, and a newer one at the same path that is + // not: the newer one is what runs. + add_script(&db, 4244, "u/alice/reused", "alice").await?; + sqlx::query( + "UPDATE script SET archived = true, extra_perms = '{\"u/operator\": false}', + created_at = now() - interval '1 day' WHERE hash = 4244", + ) + .execute(&db) + .await?; + add_script(&db, 4245, "u/alice/reused", "alice").await?; + + set_builder(&db, false, false).await?; + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/a0", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + !resp.status().is_success(), + "an operator without the builder right must not create a full-code app" + ); + + set_builder(&db, true, true).await?; + + // `*_raw_source` compiles caller-supplied sources with a bundler job on a worker. It sits + // beside `create_app_raw`, which builders MAY use, so it is the gate most likely to be opened + // by mistake later. + let resp = c + .post(format!("{api}/apps/create_raw_source")) + .json(&json!({ + "path": "u/operator/a1", "summary": "", + "value": {"files": {"index.ts": "console.log(1)"}, "runnables": {}}, + "policy": {"execution_mode": "publisher"} + })) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not compile app sources on a worker" + ); + + // Low-code apps carry inline scripts, so they stay shut too. + let resp = c + .post(format!("{api}/apps/create")) + .json(&json!({ + "path": "u/operator/a2", "summary": "", + "value": {"grid": []}, "policy": {"execution_mode": "publisher"} + })) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not create a low-code app" + ); + + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app("u/operator/a3", "publisher", "u/alice/private")) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not deploy an app referencing a runnable it cannot read" + ); + + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app("u/operator/a4", "viewer", "u/operator/some_script")) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a builder must not deploy a viewer-mode app: the policy stops bounding what it can invoke" + ); + + // Storage strips a NUL, turning this key into `inlineScript`. + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app_form(json!({ + "path": "u/operator/a_nul", "summary": "", + "value": {"files": {}, "runnables": {"r": { + "type": "inline", + "inline\u{0}Script": {"content": "x", "language": "bun"} + }}}, + "policy": {"execution_mode": "publisher", "triggerables_v2": {}} + }))) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "a NUL in a key must not hide an inline script: {}", + resp.text().await? + ); + + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/a5", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + resp.status().is_success(), + "a builder must be able to deploy a full-code app over readable runnables: {}", + resp.text().await? + ); + let sandbox: Option = sqlx::query_scalar( + "SELECT (policy->>'sandbox')::boolean FROM app WHERE workspace_id = $1 AND path = $2", + ) + .bind(WS) + .bind("u/operator/a5") + .fetch_one(&db) + .await?; + assert_eq!( + sandbox, + Some(true), + "a builder-authored app must be stored sandboxed" + ); + + // `execute_component` looks up `:` with an unrestricted component string, so + // a key with a second colon still resolves at run time and must not slip past validation. + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app_form(json!({ + "path": "u/operator/a6", "summary": "", + "value": {"files": {}, "runnables": {}}, + "policy": {"execution_mode": "publisher", "triggerables_v2": { + "x:y:script/u/alice/private": {"static_inputs": {}, "one_of_inputs": {}} + }} + }))) + .send() + .await?; + assert!( + !resp.status().is_success(), + "a multi-colon triggerable key must not hide an unreadable runnable from validation" + ); + + // The two rights are independent. + set_builder(&db, true, false).await?; + let resp = c + .post(format!("{api}/flows/create")) + .json(&composition_flow("u/operator/flows_only")) + .send() + .await?; + assert!( + resp.status().is_success(), + "flows-only must still create a flow: {}", + resp.text().await? + ); + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/flows_only_app", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + !resp.status().is_success(), + "flows-only must not deploy a full-code app" + ); + + set_builder(&db, false, true).await?; + let resp = c + .post(format!("{api}/flows/create")) + .json(&composition_flow("u/operator/apps_only")) + .send() + .await?; + assert!( + !resp.status().is_success(), + "apps-only must not create a flow" + ); + let resp = c + .post(format!("{api}/apps/create_raw")) + .multipart(raw_app( + "u/operator/apps_only_app", + "publisher", + "u/operator/some_script", + )) + .send() + .await?; + assert!( + resp.status().is_success(), + "apps-only must still deploy a full-code app: {}", + resp.text().await? + ); + + // The deploy panel saves access and frontend scopes through the settings-only endpoint. + for (body, expected) in [ + ( + json!({"policy": {"execution_mode": "anonymous", "triggerables_v2": {}}}), + 200, + ), + ( + json!({"policy": {"execution_mode": "publisher", "sandbox": false, "triggerables_v2": {}}}), + 403, + ), + ( + json!({"value": {"files": {}, "runnables": {}}, "policy": {"execution_mode": "publisher"}}), + 403, + ), + ] { + let resp = c + .post(format!("{api}/apps/update/u/operator/apps_only_app")) + .json(&body) + .send() + .await?; + let status = resp.status(); + assert_eq!( + status, + expected, + "settings update {body}: {}", + resp.text().await? + ); + } + let sandbox: Option = sqlx::query_scalar( + "SELECT (policy->>'sandbox')::boolean FROM app WHERE workspace_id = $1 AND path = $2", + ) + .bind(WS) + .bind("u/operator/apps_only_app") + .fetch_one(&db) + .await?; + assert_eq!( + sandbox, + Some(true), + "a settings update must not turn a builder app's sandbox off" + ); + + // The editor runs the runnables a draft names as whoever opens it, and its frontend code + // with their session unless the preview is sandboxed. + for (runnables, policy, expected) in [ + (json!({}), json!({"sandbox": true}), 200), + ( + json!({"a": {"type": "inline", "inlineScript": {"content": "x", "language": "bun"}}}), + json!({"sandbox": true}), + 403, + ), + (json!({}), json!({"sandbox": false}), 403), + ( + json!({}), + json!({"sandbox": true, "execution_mode": "viewer"}), + 403, + ), + (json!({}), json!(null), 403), + ( + json!({}), + json!({"sandbox": true, "on_behalf_of": "u/alice", "on_behalf_of_email": "alice@windmill.dev"}), + 403, + ), + ( + json!({}), + json!({"sandbox": true, "on_behalf_of": "u/operator"}), + 200, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables": { + format!("{}script/u/operator/some_script", "a:".repeat(300)): {} + }}), + 400, + ), + ( + json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/operator/some_script"}}), + json!({"sandbox": true}), + 200, + ), + ( + json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/alice/private"}}), + json!({"sandbox": true}), + 403, + ), + ( + json!({"r": {"type": "runnableByPath", "runType": "script", "path": "u/alice/reused"}}), + json!({"sandbox": true}), + 403, + ), + ( + json!({"r": {"type": "path", "runType": "hubscript", "path": "hub/1/x/y"}}), + json!({"sandbox": true}), + 403, + ), + ( + json!({"r": {"type": "path", "runType": "script/u/alice", "path": "u/operator/some_script"}}), + json!({"sandbox": true}), + 400, + ), + ( + json!({"r": {"type": "path", "runType": "script", "path": ["u/alice/private"]}}), + json!({"sandbox": true}), + 400, + ), + ( + json!({"r": {"type": ["path"], "runType": "script", "path": "u/alice/private"}}), + json!({"sandbox": true}), + 400, + ), + ( + json!([{"type": "path", "runType": "script", "path": "u/alice/private"}]), + json!({"sandbox": true}), + 400, + ), + ( + json!({"r": {"type": "path", "runType": "script", "path": "u/operator/some_script", + "fields": {"x": {"type": "static", "value": {"type": "path", "path": "/srv/out"}}}}}), + json!({"sandbox": true}), + 200, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables": {"x:rawscript/abc": {}}}), + 403, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables": {"x:script/u/alice/private": {}}}), + 403, + ), + ( + json!({}), + json!({"sandbox": true, "triggerables_v2": { + "x:script/u/alice/private": {"static_inputs": {}, "one_of_inputs": {}}, + "y:rawscript/abc": {"static_inputs": {}, "one_of_inputs": {}} + }}), + 200, + ), + ] { + let resp = c + .post(format!("{api}/drafts/update/raw_app/u/operator/d1")) + .json(&json!({ + "value": {"files": {}, "runnables": runnables, "summary": "", "policy": policy}, + "force": true + })) + .send() + .await?; + let status = resp.status(); + assert_eq!( + status, + expected, + "raw app draft {runnables} {policy}: {}", + resp.text().await? + ); + } + + // The component preview runs a deployed runnable for a builder; inline code and hub + // scripts are unreviewed code. + for (preview, expected) in [ + (json!({"path": "script/u/operator/some_script"}), 200), + (json!({"path": "script/hub/1/x/y"}), 403), + ( + json!({"raw_code": {"content": "x", "language": "bun"}}), + 403, + ), + ] { + let mut body = json!({"component": "c", "args": {}, "force_viewer_static_fields": {}}); + body.as_object_mut() + .unwrap() + .extend(preview.as_object().unwrap().clone()); + let resp = c + .post(format!( + "{api}/apps_u/execute_component/u/operator/apps_only_app" + )) + .json(&body) + .send() + .await?; + let status = resp.status(); + assert_eq!( + status, + expected, + "component preview {preview}: {}", + resp.text().await? + ); + } + + // Delete and redeploy each serve both kinds; builder rights cover only full-code apps. + let resp = client("SECRET_TOKEN") + .post(format!("{api}/apps/create")) + .json(&json!({ + "path": "u/operator/low_code", "summary": "", + "value": {"grid": []}, "policy": {"execution_mode": "publisher"} + })) + .send() + .await?; + assert!(resp.status().is_success(), "{}", resp.text().await?); + let resp = c + .post(format!("{api}/apps/update_raw/u/operator/low_code")) + .multipart(raw_app_form(json!({ + "policy": {"execution_mode": "publisher", "triggerables_v2": {}} + }))) + .send() + .await?; + assert_eq!( + resp.status(), + 403, + "policy-only update of a low-code app: {}", + resp.text().await? + ); + // Deployment history restores a full-code version with allow_kind_change set. + let resp = c + .post(format!("{api}/apps/update_raw/u/operator/apps_only_app")) + .multipart(raw_app_form(json!({ + "value": {"files": {}, "runnables": {}}, + "policy": {"execution_mode": "publisher", "triggerables_v2": {}}, + "allow_kind_change": true + }))) + .send() + .await?; + assert_eq!( + resp.status(), + 200, + "restore of a full-code version: {}", + resp.text().await? + ); + for (path, expected) in [("low_code", 403), ("apps_only_app", 200)] { + let resp = c + .delete(format!("{api}/apps/delete/u/operator/{path}")) + .send() + .await?; + let status = resp.status(); + assert_eq!(status, expected, "delete {path}: {}", resp.text().await?); + } + + Ok(()) +} + +/// The readability check must not hold its RLS transaction while it takes a second connection, +/// or a one-connection pool (`DATABASE_CONNECTIONS=1`) stalls every builder draft naming a script. +#[sqlx::test(migrations = "../migrations", fixtures("base", "permissions_test"))] +#[serial] +async fn test_operator_builder_app_check_fits_one_connection( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, +) -> anyhow::Result<()> { + initialize_tracing().await; + let db = pool_opts + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(5)) + .connect_with(connect_opts) + .await?; + let server = ApiServer::start(db.clone()).await?; + let api = format!("http://localhost:{}/api/w/{WS}", server.addr.port()); + add_script(&db, 4241, "u/operator/some_script", "operator").await?; + set_builder(&db, false, true).await?; + + let resp = client("OPERATOR_TOKEN_1") + .post(format!("{api}/drafts/update/raw_app/u/operator/d1")) + .json(&json!({ + "value": {"files": {}, "summary": "", "runnables": {"r": { + "name": "r", "type": "runnableByPath", "runType": "script", + "path": "u/operator/some_script" + }}, "policy": {"sandbox": true, "triggerables_v2": {}}}, + "force": true + })) + .send() + .await?; + let status = resp.status(); + assert_eq!(status, 200, "{}", resp.text().await?); + Ok(()) +} diff --git a/backend/windmill-api-jobs/src/execution.rs b/backend/windmill-api-jobs/src/execution.rs index ce6cfc185d..c6c2a1fede 100644 --- a/backend/windmill-api-jobs/src/execution.rs +++ b/backend/windmill-api-jobs/src/execution.rs @@ -245,6 +245,27 @@ pub async fn unclaimable_run_lineage( Ok(referenced) } +/// `WM_END_USER_EMAIL` of a run started through a `/jobs/run*` route. `WM_EMAIL` names the +/// runnable's `on_behalf_of` identity when it has one, so this is the only place the caller's +/// address reaches the job. A job's own `WM_TOKEN` passes on the end user of that job, and a run +/// a trigger fired has no end user. +pub async fn run_end_user_email( + db: &DB, + w_id: &str, + authed: &ApiAuthed, + trigger: Option<&TriggerMetadata>, +) -> error::Result> { + if trigger.is_some() { + return Ok(None); + } + match authed.job_id { + Some(job_id) => Ok(windmill_common::auth::get_job_perms(db, &job_id, w_id) + .await? + .and_then(|p| p.end_user_email)), + None => Ok(Some(authed.email.clone())), + } +} + #[cfg(feature = "enterprise")] pub async fn check_license_key_valid() -> error::Result<()> { use windmill_common::ee_oss::LICENSE_KEY_VALID; @@ -1013,6 +1034,7 @@ pub async fn run_flow<'c>( ) }; + let end_user_email = run_end_user_email(db, w_id, authed, trigger.as_ref()).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(db, authed).await?; let (uuid, mut tx) = push( &db, @@ -1048,7 +1070,7 @@ pub async fn run_flow<'c>( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(trigger), run_query.suspended_mode, scope_ceiling.as_deref(), @@ -1244,6 +1266,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, trigger.as_ref()).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, @@ -1277,7 +1300,7 @@ pub async fn push_script_job_by_path_into_queue<'c>( }, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(trigger), run_query.suspended_mode, scope_ceiling.as_deref(), diff --git a/backend/windmill-api-scripts/src/scripts.rs b/backend/windmill-api-scripts/src/scripts.rs index f622c4bb91..4f4c962759 100644 --- a/backend/windmill-api-scripts/src/scripts.rs +++ b/backend/windmill-api-scripts/src/scripts.rs @@ -4266,12 +4266,6 @@ async fn delete_script_by_path( check_scopes(&authed, || format!("scripts:write:{}", path))?; - if path == "u/admin/hub_sync" && w_id == "admins" { - return Err(Error::BadRequest( - "Cannot delete the global setup app".to_string(), - )); - } - if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), @@ -4445,12 +4439,6 @@ async fn delete_scripts_bulk( require_admin(authed.is_admin, &authed.username)?; - if request.paths.contains(&"u/admin/hub_sync".to_string()) && w_id == "admins" { - return Err(Error::BadRequest( - "Cannot delete the global setup app".to_string(), - )); - } - if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, AuditAuthorable::username(&authed), diff --git a/backend/windmill-api-settings/src/lib.rs b/backend/windmill-api-settings/src/lib.rs index c79b596617..116ff73457 100644 --- a/backend/windmill-api-settings/src/lib.rs +++ b/backend/windmill-api-settings/src/lib.rs @@ -30,6 +30,7 @@ mod storage_usage; use windmill_api_auth::{require_devops_role, require_super_admin, ApiAuthed}; use windmill_audit::{audit_oss::audit_log, ActionKind}; +use windmill_common::hub_resource_types::{self, SyncSource}; use windmill_common::utils::HTTP_CLIENT_PERMISSIVE as HTTP_CLIENT; use windmill_common::DB; @@ -58,10 +59,10 @@ use windmill_common::{ error::{self, pg_error_message, JsonResult, Result}, get_database_url, global_settings::{ - ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, APP_WORKSPACED_ROUTE_SETTING, - AUTOMATE_USERNAME_CREATION_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, - DEFAULT_TAGS_WORKSPACES_SETTING, DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, - EXTERNAL_INSTANCE_PG_SETTING, + ACCENT_COLOR_SETTING, AI_CONFIG_SETTING, API_BASE_URL_SETTING, + APP_WORKSPACED_ROUTE_SETTING, AUTOMATE_USERNAME_CREATION_SETTING, + CRITICAL_ALERT_MUTE_UI_SETTING, CUSTOM_TAGS_SETTING, DEFAULT_TAGS_WORKSPACES_SETTING, + DISABLE_HUB_SETTING, EMAIL_DOMAIN_SETTING, ENV_SETTINGS, EXTERNAL_INSTANCE_PG_SETTING, GITHUB_APP_WEBHOOK_BASE_URL_SETTING, HTTP_ROUTE_DEFAULT_ALLOWED_ORIGINS_SETTING, HTTP_ROUTE_WORKSPACED_ROUTE_SETTING, HUB_ACCESSIBLE_URL_SETTING, HUB_BASE_URL_SETTING, INSTANCE_BANNER_SETTING, MAX_RETENTION_OVERRIDE_WORKSPACES, @@ -1238,26 +1239,22 @@ async fn run_setting_pre_write_hook( } } } - GITHUB_APP_WEBHOOK_BASE_URL_SETTING => { - // A bad value here yields a webhook GitHub can never deliver to, and the + GITHUB_APP_WEBHOOK_BASE_URL_SETTING | API_BASE_URL_SETTING => { + // A bad webhook base yields a webhook GitHub can never deliver to, and the // failure only shows up much later as "falling back to polling" on a - // repository — so reject it at the boundary instead. + // repository; a bad API base puts an unreachable URL in front of every user. + // So reject both at the boundary instead. match value { // Clearing (delete row) is handled by the caller; allow it through. serde_json::Value::Null => {} serde_json::Value::String(s) if s.trim().is_empty() => {} serde_json::Value::String(s) => { - windmill_common::global_settings::validate_webhook_base_url(s).map_err( - |e| { - error::Error::BadRequest(format!( - "{GITHUB_APP_WEBHOOK_BASE_URL_SETTING}: {e}" - )) - }, - )?; + windmill_common::global_settings::validate_webhook_base_url(s) + .map_err(|e| error::Error::BadRequest(format!("{key}: {e}")))?; } _ => { return Err(error::Error::BadRequest(format!( - "{GITHUB_APP_WEBHOOK_BASE_URL_SETTING} must be a URL string" + "{key} must be a URL string" ))); } } @@ -1394,8 +1391,12 @@ async fn set_instance_config( // Applied and audited one setting at a time: the writes are not transactional, so a // batch that fails midway must not leave the settings it did write unrecorded. - let instance_config::SettingsDiff { upserts, deletes, mut previous_values, unchanged_count } = - settings_diff; + let instance_config::SettingsDiff { + upserts, + deletes, + mut previous_values, + unchanged_count, + } = settings_diff; let steps = upserts .into_iter() .map(|(k, v)| (k, Some(v))) @@ -2576,95 +2577,6 @@ pub async fn get_jwks() -> JsonResult { Ok(Json(JwksResponse { keys: vec![] })) } -#[derive(serde::Deserialize, serde::Serialize)] -struct CachedResourceType { - #[allow(dead_code)] - id: i64, - name: String, - schema: Option, - #[allow(dead_code)] - app: String, - description: Option, - /// Doubly optional, and read through a wrapping deserializer: this struct also - /// decodes the on-disk cache, where an absent key means "written before the - /// column, leave the stored extension alone" and an explicit null means the hub - /// dropped it. Plain serde folds both into `None`. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - format_extension: Option>, - /// Doubly optional like `format_extension`: no key leaves the stored name alone, an explicit - /// null (the hub naming nothing) clears it. - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - display_name: Option>, -} - -#[derive(serde::Deserialize)] -struct HubResourceTypeRaw { - id: i64, - name: String, - schema: Option, - app: String, - description: Option, - #[serde(default)] - format_extension: Option, - #[serde( - default, - deserialize_with = "windmill_common::more_serde::double_option" - )] - display_name: Option>, -} - -async fn fetch_resource_types_from_hub() -> error::Result> { - let response = HTTP_CLIENT - .get(format!( - "{}/resource_types/list", - windmill_common::DEFAULT_HUB_BASE_URL - )) - .header("Accept", "application/json") - .send() - .await - .map_err(|e| error::Error::InternalErr(format!("Failed to fetch from hub: {}", e)))?; - - if !response.status().is_success() { - return Err(error::Error::InternalErr(format!( - "Hub returned status {}", - response.status() - ))); - } - - let raw_types: Vec = response - .json() - .await - .map_err(|e| error::Error::InternalErr(format!("Failed to parse hub response: {}", e)))?; - - Ok(raw_types - .into_iter() - .filter_map(|rt| { - let schema = match rt.schema { - Some(s) => match serde_json::from_str(&s) { - Ok(v) => Some(v), - Err(_) => return None, - }, - None => None, - }; - Some(CachedResourceType { - id: rt.id, - name: rt.name, - schema, - app: rt.app, - description: rt.description, - format_extension: Some(rt.format_extension), - display_name: rt.display_name, - }) - }) - .collect()) -} - #[derive(serde::Deserialize)] struct SyncResourceTypesQuery { name: Option, @@ -2677,118 +2589,45 @@ async fn sync_cached_resource_types( ) -> error::Result { require_super_admin(&db, &authed).await?; - use windmill_common::worker::HUB_RT_CACHE_DIR; - let cache_path = format!("{}/resource_types.json", *HUB_RT_CACHE_DIR); + let outcome = hub_resource_types::sync(&db).await?; - // Manual sync is hub-first so it lands newly-published hub types on demand. The - // on-disk cache is only a fallback for when the hub is unreachable (airgapped - // installs / network error); refreshing it is left to the daily cache-rt cron and - // the startup sync in main.rs, which own the offline path. - let (resource_types, from_hub) = match fetch_resource_types_from_hub().await { - Ok(types) => { - tracing::info!("Fetched {} resource types live from the hub", types.len()); - (types, true) - } - Err(hub_err) => { - tracing::warn!( - "Live hub fetch failed ({hub_err}), falling back to on-disk cache at {cache_path}" - ); - match tokio::fs::read_to_string(&cache_path).await { - Ok(content) => { - let parsed = serde_json::from_str::>(&content) - .map_err(|e| { - error::Error::InternalErr(format!( - "Failed to parse cached resource types: {}", - e - )) - })?; - (parsed, false) - } - Err(_) => return Err(hub_err), - } - } - }; - - let mut synced_count = 0; - - for rt in &resource_types { - // A name too long for the column counts as absent, leaving the stored one alone: one bad - // entry must not fail the upsert and end the rest of the sync. - let display_name = match &rt.display_name { - Some(Some(name)) if name.chars().count() > 100 => None, - other => other.clone(), + if !outcome.changed.is_empty() { + let source = match outcome.source { + SyncSource::Hub => "hub", + SyncSource::ImageCache => "image_cache", }; - let exists: Option = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM resource_type WHERE workspace_id = 'admins' AND name = $1 AND schema IS NOT DISTINCT FROM $2 AND description IS NOT DISTINCT FROM $3 AND ($5 IS NOT TRUE OR format_extension IS NOT DISTINCT FROM $4) AND ($7 IS NOT TRUE OR display_name IS NOT DISTINCT FROM $6))", - &rt.name, - rt.schema.as_ref(), - rt.description.as_deref(), - rt.format_extension.clone().flatten(), - rt.format_extension.is_some(), - display_name.clone().flatten(), - display_name.is_some(), + let changed = outcome.changed.join(", "); + audit_log( + &db, + &authed, + "resource_types.sync", + ActionKind::Update, + "admins", + None, + Some([("source", source), ("changed", changed.as_str())].into()), ) - .fetch_one(&db) .await?; - - if exists.unwrap_or(false) { - continue; - } - - sqlx::query!( - // Whether the payload carried the key at all is what decides: present - // (even as null) is authoritative and may clear, absent means a cache - // written before the column and must leave the stored value alone. - "INSERT INTO resource_type (workspace_id, name, schema, description, format_extension, display_name, edited_at) - VALUES ('admins', $1, $2, $3, $4, $6, now()) - ON CONFLICT (workspace_id, name) DO UPDATE - SET schema = EXCLUDED.schema, description = EXCLUDED.description, - -- A fileset is a set of files, so it cannot also be one file. - -- Create and update reject the pair; this writer bypasses both, so - -- it declines the extension rather than persisting the forbidden - -- combination onto a same-named local fileset. - format_extension = CASE - WHEN resource_type.is_fileset THEN NULL - WHEN $5 THEN EXCLUDED.format_extension - ELSE resource_type.format_extension END, - display_name = CASE WHEN $7 THEN EXCLUDED.display_name ELSE resource_type.display_name END, - edited_at = now()", - &rt.name, - rt.schema.as_ref(), - rt.description.as_deref(), - rt.format_extension.clone().flatten(), - rt.format_extension.is_some(), - display_name.clone().flatten(), - display_name.is_some(), - ) - .execute(&db) - .await?; - - synced_count += 1; } // If a specific type was requested and is still absent after syncing, surface an // explicit not-found instead of a silent "Synced 0". Word it by source so the // cache-fallback path does not claim it checked the hub. if let Some(name) = name.as_deref() { - if !resource_types.iter().any(|rt| rt.name == name) { - let source = if from_hub { - "on the hub" - } else { - "in the cached resource types (hub unreachable)" + if !outcome.listed.iter().any(|listed| listed == name) { + let source = match outcome.source { + SyncSource::Hub => "on the hub", + SyncSource::ImageCache => "in the cached resource types (hub unavailable)", }; return Err(error::Error::NotFound(format!( - "resource type '{}' not found {}", - name, source + "resource type '{name}' not found {source}" ))); } } - Ok(format!( - "Synced {} resource types ({} unchanged)", - synced_count, - resource_types.len() - synced_count - )) + Ok(match outcome.source { + SyncSource::Hub => outcome.summary(), + SyncSource::ImageCache => format!("{}, the hub being unavailable", outcome.summary()), + }) } #[cfg(test)] diff --git a/backend/windmill-api-users/src/users.rs b/backend/windmill-api-users/src/users.rs index ac9defcd53..46bd6825e8 100644 --- a/backend/windmill-api-users/src/users.rs +++ b/backend/windmill-api-users/src/users.rs @@ -490,7 +490,7 @@ async fn list_user_usage( " SELECT usr.email, usage.executions FROM usr, LATERAL ( - SELECT COALESCE(SUM(c.duration_ms + 1000)/1000 , 0)::BIGINT executions + SELECT COALESCE(SUM(COALESCE((c.extras->>'wac_last_segment_ms')::bigint, c.duration_ms) + 1000)/1000 , 0)::BIGINT executions FROM v2_job_completed c JOIN v2_job j USING (id) WHERE j.workspace_id = $1 AND j.kind NOT IN ('flow', 'flowpreview', 'flownode', 'singlestepflow') diff --git a/backend/windmill-api-workers/src/lib.rs b/backend/windmill-api-workers/src/lib.rs index cf0b85609a..8ebee2a967 100644 --- a/backend/windmill-api-workers/src/lib.rs +++ b/backend/windmill-api-workers/src/lib.rs @@ -41,6 +41,7 @@ pub fn global_service() -> Router { .route("/queue_metrics", get(get_queue_metrics)) .route("/queue_metrics_series", get(get_queue_metrics_series)) .route("/queue_status", get(get_queue_status)) + .route("/runnable_stats", get(get_runnable_stats)) .route("/queue_counts", get(get_queue_counts)) .route("/queue_running_counts", get(get_queue_running_counts)) .route( @@ -299,6 +300,81 @@ async fn get_queue_metrics( Ok(Json(queue_metrics)) } +#[derive(Deserialize)] +struct RunnableStatsQuery { + window_secs: Option, + worker_group: Option, + workspace: Option, + order_by: Option, + limit: Option, +} + +#[derive(Serialize)] +struct RunnableStats { + workspace_id: String, + runnable_path: String, + worker_group: String, + job_count: i64, + total_duration_ms: i64, + max_memory_peak: i32, + sum_memory_peak: i64, + memory_sample_count: i64, + total_cpu_ms: i64, +} + +async fn get_runnable_stats( + authed: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult> { + require_devops_role(&db, &authed).await?; + + let window = query.window_secs.unwrap_or(24 * 3600).clamp( + 3600, + windmill_common::runnable_job_stats::RETENTION_DAYS * 24 * 3600, + ); + // Rows are hourly buckets keyed by their start, so the bucket the window begins in + // is included whole. + let from_hour = (chrono::Utc::now().timestamp() - window) / 3600 * 3600; + let order_by = match query.order_by.as_deref() { + Some("max_memory") => "max_memory", + Some("cpu") => "cpu", + _ => "total_duration", + }; + + let rows = sqlx::query_as!( + RunnableStats, + r#" + SELECT workspace_id, runnable_path, worker_group, + SUM(job_count)::bigint AS "job_count!", + SUM(total_duration_ms)::bigint AS "total_duration_ms!", + MAX(max_memory_peak) AS "max_memory_peak!", + SUM(sum_memory_peak)::bigint AS "sum_memory_peak!", + SUM(memory_sample_count)::bigint AS "memory_sample_count!", + SUM(total_cpu_ms)::bigint AS "total_cpu_ms!" + FROM runnable_job_stats + WHERE hour >= $1 + AND ($2::text IS NULL OR worker_group = $2) + AND ($3::text IS NULL OR workspace_id = $3) + GROUP BY workspace_id, runnable_path, worker_group + ORDER BY CASE $4::text + WHEN 'max_memory' THEN MAX(max_memory_peak)::numeric + WHEN 'cpu' THEN SUM(total_cpu_ms) + ELSE SUM(total_duration_ms) + END DESC, workspace_id, runnable_path, worker_group + LIMIT $5 + "#, + from_hour, + query.worker_group, + query.workspace, + order_by, + query.limit.unwrap_or(100).clamp(1, 1000), + ) + .fetch_all(&db) + .await?; + Ok(Json(rows)) +} + #[derive(Deserialize)] struct QueueMetricsSeriesQuery { window_secs: Option, diff --git a/backend/windmill-api-workspaces/src/workspaces.rs b/backend/windmill-api-workspaces/src/workspaces.rs index efc108453d..7f6eace660 100644 --- a/backend/windmill-api-workspaces/src/workspaces.rs +++ b/backend/windmill-api-workspaces/src/workspaces.rs @@ -11608,10 +11608,13 @@ struct ChangeOperatorSettings { /// withdraw or grant the right on every push. /// /// `builder_flows` lets every operator of this workspace compose flows out of already-deployed - /// runnables, and makes each of them consume a full author seat instead of half of one. + /// runnables, `builder_apps` full-code apps. Either makes each operator consume a full author + /// seat instead of half of one. #[serde(default, skip_serializing_if = "Option::is_none")] builder_flows: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + builder_apps: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] manage_schedules: Option, #[serde(default, skip_serializing_if = "Option::is_none")] manage_triggers: Option, @@ -11626,9 +11629,10 @@ async fn update_operator_settings( require_admin(authed.is_admin, &authed.username)?; // Every operator of the workspace turns into a full seat, which an offline license may not - // cover. It is a no-op delta when the right is already on. + // cover. Either right costs the same seat, so this prices "any", and it is a no-op delta when + // one of them is already on. #[cfg(feature = "enterprise")] - if settings.builder_flows == Some(true) { + if settings.builder_flows == Some(true) || settings.builder_apps == Some(true) { if let Some(msg) = windmill_common::ee_oss::check_seat_cap_for_operator_builder(&db, &w_id).await? { diff --git a/backend/windmill-api-workspaces/src/workspaces_extra.rs b/backend/windmill-api-workspaces/src/workspaces_extra.rs index 50833bfe06..2f0d4deb13 100644 --- a/backend/windmill-api-workspaces/src/workspaces_extra.rs +++ b/backend/windmill-api-workspaces/src/workspaces_extra.rs @@ -1299,6 +1299,13 @@ pub(crate) async fn delete_workspace( .execute(&mut *tx) .await?; + sqlx::query!( + "DELETE FROM runnable_job_stats WHERE workspace_id = $1", + &w_id + ) + .execute(&mut *tx) + .await?; + // If this workspace is itself a dev workspace, deleting it dissolves the pairing, so also drop // the parent prod's reserved dev_workspace_lock (mirrors detach_dev_workspace) — otherwise prod // stays locked against direct deploy/forking with no dev workspace left to make changes in. diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 5e635e39bd..bab02327ec 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.828.0 + version: 1.830.0 title: Windmill API contact: @@ -1919,7 +1919,7 @@ paths: /settings/instance_ui: get: - summary: get the instance-wide announcement banner and accent color + summary: get the instance-wide announcement banner, accent color and API base url description: | Readable by any authenticated user; every page load reads it. Each value is the raw stored setting, or null when unset. @@ -1928,7 +1928,7 @@ paths: - setting responses: "200": - description: the instance_banner and accent_color settings + description: the instance_banner, accent_color and api_base_url settings content: application/json: schema: @@ -1936,6 +1936,7 @@ paths: properties: instance_banner: {} accent_color: {} + api_base_url: {} /settings_u/ruff_config: get: @@ -10924,6 +10925,37 @@ paths: description: "`saved` only, upsert or delete: where the write landed. Differs from the URL path when the item had moved away from it; the editor follows it there. Absent when a delete found nothing to remove and the caller cannot read the path it moved to." required: [status, current_timestamp] + /w/{workspace}/drafts/delete_for_user/{kind}/{path}: + post: + summary: delete another user's draft, or every draft at a path (admin only) + description: Delete the draft a given user holds at a path, or, without `username`, every draft at that path (the legacy ownerless one and the caller's own included). Workspace admins / superadmins only, and only for script, flow, app and raw_app drafts. Unlike a discard of one's own draft through `drafts/update`, this returns 404 when nothing was deleted. + operationId: deleteDraftForUser + tags: + - draft + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - name: kind + in: path + required: true + schema: + $ref: "#/components/schemas/UserDraftItemKind" + - $ref: "#/components/parameters/ScriptPath" + - name: username + in: query + required: false + description: Workspace username of the draft owner. Omit to delete every user's draft at the path. + schema: + type: string + responses: + "200": + description: draft(s) deleted + content: + text/plain: + schema: + type: string + "404": + description: no matching draft at that path + /w/{workspace}/drafts/move/{kind}/{path}: post: summary: move the current user's draft-only item to another path @@ -23523,6 +23555,54 @@ paths: - id - values + /workers/runnable_stats: + get: + summary: get resource usage per runnable and worker group over a time window (require devops role) + operationId: getRunnableStats + tags: + - worker + parameters: + - name: window_secs + in: query + required: false + description: how far back to read, in seconds (defaults to one day, capped at the 30-day retention) + schema: + type: integer + - name: worker_group + in: query + required: false + description: only the jobs run by this worker group + schema: + type: string + - name: workspace + in: query + required: false + description: only the jobs of this workspace + schema: + type: string + - name: order_by + in: query + required: false + description: "what the rows are ranked by, descending (default: total_duration)" + schema: + type: string + enum: [total_duration, max_memory, cpu] + - name: limit + in: query + required: false + description: number of rows to return (default 100, at most 1000) + schema: + type: integer + responses: + "200": + description: resource usage per runnable and worker group, heaviest first + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/RunnableStats" + /workers/queue_metrics_series: get: summary: get the queue metrics of a time window, as a bounded line per tag @@ -25230,9 +25310,12 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": - description: Parquet Preview + description: > + Parquet Preview: `columns`, `column_types` (Arrow type names) and `rows`. content: application/json: {} @@ -25257,6 +25340,13 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" + - name: csv_separator + in: query + schema: + type: string + - $ref: "#/components/parameters/TablePreviewCsvHasHeader" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": description: Table count @@ -25305,13 +25395,20 @@ paths: in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewS3ResourcePath" - name: csv_separator in: query schema: type: string + - $ref: "#/components/parameters/TablePreviewCsvHasHeader" + - $ref: "#/components/parameters/TablePreviewFilters" responses: "200": - description: Csv Preview + description: > + Csv Preview: `columns`, `column_types` (Arrow type names), `rows`, and the + `csv_separator` the file was read with, guessed from its first bytes when + none is given. + A `.jsonl` or `.ndjson` file is read as newline-delimited JSON. content: application/json: {} @@ -27671,6 +27768,28 @@ components: required: true schema: type: string + TablePreviewS3ResourcePath: + name: s3_resource_path + description: Read the file from this object storage resource instead of the workspace storage. + in: query + schema: + type: string + TablePreviewCsvHasHeader: + name: csv_has_header + description: Whether the first CSV line names the columns. Defaults to true. + in: query + schema: + type: boolean + TablePreviewFilters: + name: filters + description: > + JSON object, column name to filter. A number column takes an optional comparison + operator and a number (`>=10`), a boolean column a boolean, and any other column a + case-insensitive substring, or `=value` for an exact match. A `search_term` without + a `search_col` matches any column, case-insensitively. + in: query + schema: + type: string HubPublishFolder: name: folder in: query @@ -31731,6 +31850,7 @@ components: - "resource_types.create" - "resource_types.update" - "resource_types.delete" + - "resource_types.sync" - "schedule.create" - "schedule.setenabled" - "schedule.edit" @@ -35874,6 +35994,45 @@ components: `u/`, `g/`, or an email that exists in this workspace. + RunnableStats: + type: object + description: > + Resource usage of the jobs of one runnable on one worker group. Memory and CPU are + only known for jobs run as a subprocess; memory is the peak of the job's main process. + properties: + workspace_id: + type: string + runnable_path: + type: string + description: path of the script or flow step; `` gathers previews and other runs without a stable path + worker_group: + type: string + job_count: + type: integer + total_duration_ms: + type: integer + max_memory_peak: + type: integer + description: highest peak memory of a job, in kB + sum_memory_peak: + type: integer + description: sum of the peak memory of the jobs that have one, in kB + memory_sample_count: + type: integer + description: number of jobs that have a peak memory reading + total_cpu_ms: + type: integer + required: + - workspace_id + - runnable_path + - worker_group + - job_count + - total_duration_ms + - max_memory_peak + - sum_memory_peak + - memory_sample_count + - total_cpu_ms + WorkerPing: type: object properties: @@ -37887,6 +38046,9 @@ components: builder_flows: type: boolean description: Whether operators can compose flows out of existing runnables (consumes a full seat). Omitting the field leaves the stored value unchanged. + builder_apps: + type: boolean + description: Whether operators can compose full-code apps out of existing runnables (consumes a full seat). Omitting the field leaves the stored value unchanged. manage_schedules: type: boolean description: Whether operators can create, edit and delete schedules. Granted unless withdrawn; omitting the field leaves the stored value unchanged. diff --git a/backend/windmill-api/src/ai.rs b/backend/windmill-api/src/ai.rs index c401568868..cf7447ceba 100644 --- a/backend/windmill-api/src/ai.rs +++ b/backend/windmill-api/src/ai.rs @@ -40,6 +40,7 @@ use windmill_ai::proxy::{ fim::maybe_transform_fim_request, proxy_execution_mode, ProxyBuildArgs, ProxyExecutionMode, ProxyRequest, }; +use windmill_ai::retry::send_with_retries; use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::db::UserDB; use windmill_common::error::{to_anyhow, Error, Result}; @@ -1188,15 +1189,11 @@ async fn global_proxy( } }; - let response = request.send().await.map_err(to_anyhow)?; + let response = send_with_retries(request).await; audit_global_ai_request(&db, &authed).await?; - if response.error_for_status_ref().is_err() { - let err_msg = response.text().await.unwrap_or("".to_string()); - return Err(Error::AIError(err_msg)); - } - + let response = response?; let status_code = response.status(); let headers = response.headers().clone(); let stream = response.bytes_stream(); @@ -1576,7 +1573,7 @@ async fn proxy( } }; - let response = request.send().await.map_err(to_anyhow)?; + let response = send_with_retries(request).await; let mut tx = db.begin().await?; @@ -1592,11 +1589,7 @@ async fn proxy( .await?; tx.commit().await?; - if response.error_for_status_ref().is_err() { - let err_msg = response.text().await.unwrap_or("".to_string()); - return Err(Error::AIError(err_msg)); - } - + let response = response?; let status_code = response.status(); let headers = response.headers().clone(); let is_sse = is_sse_response(&headers); diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 26b369ce74..430aa801a0 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -56,7 +56,10 @@ use std::str; use windmill_audit::audit_oss::{audit_log, AuditAuthorable}; use windmill_audit::ActionKind; use windmill_common::{ - apps::{AppScriptId, ListAppQuery, APP_WORKSPACED_ROUTE}, + apps::{ + app_value_has_inline_script, app_value_runnable_paths, AppScriptId, ListAppQuery, + APP_WORKSPACED_ROUTE, + }, auth::{APP_EMBED_TOKEN_LABEL_PREFIX, RAW_APP_SDK_TOKEN_LABEL_PREFIX, TOKEN_PREFIX_LEN}, cache::{self, future::FutureCachedExt}, db::{DbWithOptAuthed, UserDB}, @@ -75,7 +78,8 @@ use windmill_common::{ variables::{build_crypt, build_crypt_with_key_suffix, encrypt}, worker::{to_raw_value, CLOUD_HOSTED}, workspaces::{ - check_deploy_rules, check_user_against_rule, ProtectionRuleKind, RuleCheckResult, + check_deploy_rules, check_operator_can_build, check_user_against_rule, + operator_builder_rights, BuilderKind, ProtectionRuleKind, RuleCheckResult, }, HUB_BASE_URL, }; @@ -749,7 +753,7 @@ async fn list_apps( // Append the authed user's `app`/`raw_app` drafts at paths with no deployed app; // see scripts.rs. if lq.include_draft_only.unwrap_or(false) - && !authed.is_operator + && (!authed.is_operator || operator_builder_rights(&db, &w_id).await?.apps) && offset == 0 && lq.path_start.is_none() && lq.path_exact.is_none() @@ -1740,11 +1744,14 @@ async fn mint_preview_sdk_token( Path(w_id): Path, Json(req): Json, ) -> Result { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot preview raw apps".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "preview raw apps", + ) + .await?; check_scopes(&authed, || format!("apps:write:{}", req.path))?; let (token, _expiration) = mint_raw_app_sdk_token(&db, &w_id, &req.path, &authed, &req.scopes, job_id).await?; @@ -2102,18 +2109,23 @@ async fn get_id_from_secret( prefix: Option<&str>, ) -> Result { let mc = build_crypt(db, w_id).await?; - let decrypted = mc - .decrypt_bytes_to_bytes(&(hex::decode(secret)?)) - .map_err(|e| Error::internal_err(e.to_string()))?; - let mut bytes = str::from_utf8(&decrypted).map_err(to_anyhow)?; - if let Some(prefix) = prefix { - if !bytes.starts_with(prefix) { - return Err(Error::BadRequest("Invalid secret".to_string())); - } - bytes = bytes.strip_prefix(prefix).unwrap_or(""); + id_from_secret(&mc, &secret, prefix) + .ok_or_else(|| Error::BadRequest("Invalid secret".to_string())) +} + +/// The secret comes from the URL of unauthenticated routes, so every way it can be invalid +/// must produce the same response: a caller must never learn why a secret was rejected. +fn id_from_secret( + mc: &magic_crypt::MagicCrypt256, + secret: &str, + prefix: Option<&str>, +) -> Option { + let decrypted = mc.decrypt_bytes_to_bytes(&hex::decode(secret).ok()?).ok()?; + let plaintext = String::from_utf8(decrypted).ok()?; + match prefix { + Some(prefix) => plaintext.strip_prefix(prefix)?.parse().ok(), + None => plaintext.parse().ok(), } - let id: i64 = bytes.parse().map_err(to_anyhow)?; - Ok(id) } async fn get_public_resource( @@ -2262,6 +2274,260 @@ async fn store_raw_app_file<'a>( Ok(()) } +/// A builder app write is held to one set of rules whether it deploys or saves a draft: the +/// editor previews a draft as whoever opens it, and an admin may deploy it as it stands. A deploy +/// fills in what a draft must state, since it stores the policy it checked. +#[derive(Clone, Copy, PartialEq)] +pub(crate) enum BuilderAppWrite { + Deploy, + Draft, +} + +/// The half of the builder-app check that needs no DB (see `docs/operator-builder-rights.md`). +/// Returns every runnable the app can invoke, as `(is_flow, path)`, from both the policy's +/// triggerables and the value's path runnables: they are not the same list. +fn check_operator_composed_app( + write: BuilderAppWrite, + raw_app: bool, + value: Option<&RawValue>, + policy: Option<&mut Policy>, +) -> Result> { + let verb = if write == BuilderAppWrite::Deploy { + "deploy" + } else { + "save" + }; + if !raw_app { + return Err(Error::PermissionDenied( + "Operators with builder rights can only author full-code apps".to_string(), + )); + } + let mut referenced: Vec<(bool, String)> = Vec::new(); + if let Some(value) = value { + // The text stored, NULs stripped: a stripped NUL can rename a key into one this reads. + let value: serde_json::Value = + serde_json::from_str(&strip_json_nul(value.get())).map_err(to_anyhow)?; + if app_value_has_inline_script(&value) { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights cannot {verb} an app carrying inline scripts" + ))); + } + referenced.extend(app_value_runnable_paths(&value)?); + } + let Some(policy) = policy else { + return Err(match write { + BuilderAppWrite::Deploy => Error::BadRequest( + "Operators with builder rights must deploy an app with its policy".to_string(), + ), + BuilderAppWrite::Draft => Error::PermissionDenied( + "Operators with builder rights can only save sandboxed apps".to_string(), + ), + }); + }; + // Every deploy of a draft rebuilds `triggerables_v2` from the runnables, so a draft's copy, + // stale since the last deploy, must not block autosave. A deploy path that skips the rebuild + // must stop exempting it here. + if write == BuilderAppWrite::Draft { + policy.triggerables_v2 = None; + } + referenced.extend(checked_policy_runnable_paths(policy)?); + match (policy.sandbox, write) { + (Some(true), _) => {} + (None, BuilderAppWrite::Deploy) => policy.sandbox = Some(true), + _ => { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights can only {verb} sandboxed apps" + ))) + } + } + + // Viewer mode would make the triggerables collected above non-exhaustive (see the docs), and + // the deploy panel shows it as members-only, so an admin deploying a draft would not see it. + // Pin an omitted mode on deploy: `update_app_internal` resolves it to the deployed app's, so a + // redeploy over an admin's viewer-mode app would otherwise inherit `Viewer`. + match policy.stated_execution_mode() { + Some(ExecutionMode::Viewer) => { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights cannot {verb} an app that runs as its viewer" + ))) + } + None if write == BuilderAppWrite::Deploy => { + policy.set_execution_mode(ExecutionMode::Publisher) + } + _ => {} + } + + referenced.sort(); + referenced.dedup(); + refuse_hub_runnables(&referenced)?; + Ok(referenced) +} + +/// The runnables a builder app's policy authorizes, as `(is_flow, path)`; refuses a policy that +/// pins inline code. +fn checked_policy_runnable_paths(policy: &Policy) -> Result> { + // A `rawscript/` triggerable is the deployed app's authorization to run caller-supplied + // `raw_code` hashing to it: pinning one hands a builder arbitrary code execution through an + // app whose value passed the inline-script check. A composition-only app has none. + fn pins_raw_script(map: &Option>) -> bool { + map.iter() + .flat_map(|m| m.keys()) + .any(|k| k.starts_with("rawscript/") || k.contains(":rawscript/")) + } + if pins_raw_script(&policy.triggerables) || pins_raw_script(&policy.triggerables_v2) { + return Err(Error::PermissionDenied( + "Operators with builder rights cannot save or deploy an app whose policy pins inline code" + .to_string(), + )); + } + // `execute_component` looks up `{component}:{path}` with an unrestricted component, so every + // colon is a possible split (`a:b:script/x` resolves for `component = "a:b"`): check every + // suffix that parses as a runnable. That copies a key once per colon, so bound the key first: + // a 255-char path plus a component name is far below the cap. + const MAX_KEY_LEN: usize = 512; + let keys = || { + policy + .triggerables + .iter() + .flat_map(|t| t.keys()) + .chain(policy.triggerables_v2.iter().flat_map(|t| t.keys())) + }; + if keys().any(|k| k.len() > MAX_KEY_LEN) { + return Err(Error::BadRequest(format!( + "App policy triggerable keys must be at most {MAX_KEY_LEN} bytes" + ))); + } + Ok(keys() + .flat_map(|key| { + std::iter::once(key.as_str()) + .chain(key.match_indices(':').map(|(i, _)| &key[i + 1..])) + .filter_map(|candidate| { + candidate + .strip_prefix("script/") + .map(|p| (false, p.to_string())) + .or_else(|| { + candidate + .strip_prefix("flow/") + .map(|p| (true, p.to_string())) + }) + }) + .collect::>() + }) + .collect()) +} + +fn refuse_hub_runnables(referenced: &[(bool, String)]) -> Result<()> { + for (_, path) in referenced { + if path.starts_with("hub/") { + return Err(Error::PermissionDenied(format!( + "Operators with builder rights cannot reference the hub runnable {path}. Deploy it to the workspace first." + ))); + } + } + Ok(()) +} + +/// Runs on every app write by an operator with builder rights, from inside the create/update +/// internals so both raw-app endpoints are covered by one check. +/// +/// `execute_component` resolves the runnable it picks with the root DB handle, so a path the +/// builder cannot read would still run, and in a `Viewer` app it would run as whoever opened it. +/// RLS on this transaction is the check; refusing `Viewer` above is what makes it exhaustive. +pub(crate) async fn validate_operator_composed_app( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + write: BuilderAppWrite, + raw_app: bool, + value: Option<&RawValue>, + policy: Option<&mut Policy>, +) -> Result<()> { + // A deploy resets the run identity to its builder (unless they are in `wm_deployers`, see + // docs/operator-builder-rights.md); a draft keeps whatever it names, which an admin's + // one-click draft deploy preserves without showing it. + if let (BuilderAppWrite::Draft, Some(policy)) = (write, policy.as_deref()) { + let names_another = policy + .on_behalf_of + .as_deref() + .is_some_and(|p| p != username_to_permissioned_as(&authed.username)) + || policy + .on_behalf_of_email + .as_deref() + .is_some_and(|e| e != authed.email); + if names_another { + return Err(Error::PermissionDenied( + "Operators with builder rights cannot save an app that runs as someone else" + .to_string(), + )); + } + } + let referenced = check_operator_composed_app(write, raw_app, value, policy)?; + require_runnables_readable(authed, db, user_db, w_id, referenced).await +} + +async fn require_runnables_readable( + authed: &ApiAuthed, + db: &DB, + user_db: &UserDB, + w_id: &str, + referenced: Vec<(bool, String)>, +) -> Result<()> { + if referenced.is_empty() { + return Ok(()); + } + // Each script version carries its own grants, and a path reused after its versions were + // archived starts with none. Check the version a run would pick, found without RLS, not + // whichever version the caller happens to see. Resolved before the transaction opens so no + // request holds two pool connections at once. + let mut hashes = Vec::with_capacity(referenced.len()); + for (is_flow, path) in &referenced { + hashes.push(if *is_flow { + None + } else { + windmill_common::get_latest_script_hash(db, path, w_id).await? + }); + } + let mut tx = user_db.clone().begin(authed).await?; + for ((is_flow, path), hash) in referenced.into_iter().zip(hashes) { + let readable = if is_flow { + sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM flow WHERE workspace_id = $1 AND path = $2)", + w_id, + path, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false) + } else { + match hash { + Some(hash) => sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM script WHERE workspace_id = $1 AND hash = $2)", + w_id, + hash, + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false), + None => false, + } + }; + // One message for every cause: telling "no runnable version" apart from "not readable" + // would reveal that a path the caller cannot read exists. + if !readable { + return Err(Error::PermissionDenied(if is_flow { + format!("Flow {path} does not exist or is not readable by you") + } else { + format!( + "Script {path} does not exist, is not readable by you, or has not finished deploying" + ) + })); + } + } + tx.commit().await?; + Ok(()) +} + macro_rules! process_app_multipart { ($authed:expr, $user_db:expr, $db:expr, $w_id:expr, $path:expr, $multipart:expr, $internal_fn:expr) => { async { @@ -2338,11 +2604,14 @@ async fn create_app_raw<'a>( Path(w_id): Path, multipart: Multipart, ) -> Result<(StatusCode, String)> { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot create apps for security reasons".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "create apps", + ) + .await?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, @@ -2519,6 +2788,19 @@ async fn create_app_internal<'a>( if raw_app { validate_raw_app_path_keys(&app.value.0)?; } + if authed.is_operator { + validate_operator_composed_app( + &authed, + &db, + &user_db, + w_id, + BuilderAppWrite::Deploy, + raw_app, + Some(&app.value.0), + Some(&mut app.policy), + ) + .await?; + } if *CLOUD_HOSTED { let nb_apps = sqlx::query_scalar!("SELECT COUNT(*) FROM app WHERE workspace_id = $1", &w_id) @@ -2825,13 +3107,23 @@ async fn delete_app( Extension(webhook): Extension, Path((w_id, path)): Path<(String, StripPath)>, ) -> Result { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot delete apps for security reasons".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "delete apps", + ) + .await?; let path = path.to_path(); check_scopes(&authed, || format!("apps:write:{}", path))?; + // One endpoint deletes both kinds, and builder rights only cover full-code apps. + if authed.is_operator && deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true) + { + return Err(Error::PermissionDenied( + "Operators with builder rights can only delete full-code apps".to_string(), + )); + } if path == "g/all/setup_app" && w_id == "admins" { return Err(Error::BadRequest( @@ -3019,13 +3311,20 @@ async fn update_app( Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, ) -> JsonResult { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot update apps for security reasons".to_string(), - )); - } - // create_app_internal(authed, user_db, db, &w_id, &mut app).await?; let path = path.to_path(); + // The deploy panel saves a deployed full-code app's access and frontend scopes here. A + // builder may change those settings; the app itself only through `update_app_raw`. + if authed.is_operator { + check_operator_can_build(&db, &w_id, true, BuilderKind::Apps, "update apps").await?; + if ns.value.is_some() + || deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true) + { + return Err(Error::PermissionDenied( + "Operators with builder rights can only change the settings of full-code apps" + .to_string(), + )); + } + } check_scopes(&authed, || format!("apps:write:{}", path))?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( @@ -3042,8 +3341,10 @@ async fn update_app( let opath = path.to_string(); let db2 = db.clone(); + // Settings-only for an operator, of an app checked above to be full-code. + let raw_app = authed.is_operator; let (new_tx, npath, v_id) = - update_app_internal(authed, db, user_db, &w_id, path, false, ns, None).await?; + update_app_internal(authed, db, user_db, &w_id, path, raw_app, ns, None).await?; new_tx.commit().await?; tally_app_rename(&db2, &w_id, &opath, &npath, v_id).await; @@ -3072,6 +3373,10 @@ async fn update_app_raw_source( Path((w_id, path)): Path<(String, StripPath)>, Json(ns): Json, ) -> JsonResult { + // Stays closed to operators even with builder rights, unlike `create_app_raw`/`update_app_raw` + // next to it: this path compiles caller-supplied sources with a bundler job on a worker, which + // is arbitrary code execution. Builders lose nothing: the browser and the CLI both bundle + // locally and deploy through the multipart endpoints. if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot update apps for security reasons".to_string(), @@ -3232,6 +3537,7 @@ async fn create_app_raw_source( Path(w_id): Path, Json(mut app): Json, ) -> Result<(StatusCode, String)> { + // Closed to builders too, for the reason on `update_app_raw_source`. if authed.is_operator { return Err(Error::NotAuthorized( "Operators cannot create apps for security reasons".to_string(), @@ -3397,11 +3703,14 @@ async fn update_app_raw<'a>( Path((w_id, path)): Path<(String, StripPath)>, multipart: Multipart, ) -> JsonResult { - if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot update apps for security reasons".to_string(), - )); - } + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Apps, + "update apps", + ) + .await?; if let RuleCheckResult::Blocked(msg) = check_deploy_rules( &w_id, @@ -3417,6 +3726,14 @@ async fn update_app_raw<'a>( let path = path.to_path(); check_scopes(&authed, || format!("apps:write:{}", path))?; + // `update_app_internal` only compares kinds when a new value is deployed, so a + // policy-only update would otherwise let a builder rewrite a low-code app's policy. + if authed.is_operator && deployed_app_kind(&user_db, &authed, &w_id, path).await? != Some(true) + { + return Err(Error::PermissionDenied( + "Operators with builder rights can only update full-code apps".to_string(), + )); + } let opath = path.to_string(); let db2 = db.clone(); let (npath, v_id) = process_app_multipart!( @@ -3484,6 +3801,23 @@ async fn update_app_internal<'a>( } } + if authed.is_operator { + // Restoring a full-code version always sends allow_kind_change. A builder converts + // nothing, so drop it and let the locked kind check below refuse an actual conversion. + ns.allow_kind_change = None; + validate_operator_composed_app( + &authed, + &db, + &user_db, + w_id, + BuilderAppWrite::Deploy, + raw_app, + ns.value.as_ref().map(|v| v.0.as_ref()), + ns.policy.as_mut(), + ) + .await?; + } + // Resolved on the (non-RLS) pool before the RLS transaction opens, for the reason // `create_app` states, with the same known, accepted rename race (see // `resolve_on_behalf_of`). Submitting a policy is how a deployer claims the app's execution @@ -4140,10 +4474,26 @@ async fn execute_component( let authed = opt_authed.as_ref().ok_or_else(|| { Error::NotAuthorized("App component preview requires authentication".to_string()) })?; + // A builder testing the app it is composing only ever previews a deployed runnable + // (`path`), confined below to what it may read. Inline `raw_code`, which an `id` also + // requires, is authoring code, so it stays closed to every operator. So is a hub + // path, which `require_path_read_access_for_preview` admits for everyone and + // `get_payload_tag_from_prefixed_path` then downloads and enqueues: it is exactly the + // unreviewed code the composition check refuses in a flow. + let previews_hub = payload.path.as_deref().is_some_and(|p| { + p.strip_prefix("script/") + .or_else(|| p.strip_prefix("flow/")) + .unwrap_or(p) + .starts_with("hub/") + }); if authed.is_operator { - return Err(Error::NotAuthorized( - "Operators cannot run preview jobs for security reasons".to_string(), - )); + let msg = "Operators cannot run preview jobs for security reasons".to_string(); + if !operator_builder_rights(&db, &w_id).await?.apps { + return Err(Error::NotAuthorized(msg)); + } + if payload.raw_code.is_some() || previews_hub { + return Err(Error::PermissionDenied(msg)); + } } check_scopes(authed, || format!("jobs:run"))?; if let Some(p) = payload.path.as_deref() { @@ -5498,6 +5848,10 @@ impl AppLoadCountQuery { search_col: self.search_col, search_term: self.search_term, storage: self.storage, + s3_resource_path: None, + csv_separator: None, + csv_has_header: None, + filters: None, }, ) } @@ -5531,6 +5885,9 @@ impl AppLoadPreviewQuery { search_term: self.search_term, storage: self.storage, csv_separator: self.csv_separator, + s3_resource_path: None, + csv_has_header: None, + filters: None, }, ) } @@ -5618,7 +5975,9 @@ async fn app_load_table_count( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = - crate::job_helpers_oss::load_table_count_internal(job_authed, &db, &w_id, file_key, inner) + crate::job_helpers_oss::load_table_count_internal( + job_authed, &db, None, &w_id, file_key, inner, + ) .await?; Ok(Json(resp).into_response()) } @@ -5637,7 +5996,7 @@ async fn app_load_parquet_preview( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( - job_authed, &db, &w_id, file_key, inner, true, + job_authed, &db, None, &w_id, file_key, inner, true, ) .await?; Ok(Json(resp).into_response()) @@ -5657,7 +6016,7 @@ async fn app_load_csv_preview( let job_authed = app_s3_on_behalf_and_provenance(&db, &path, &w_id, &opt_authed, &file_query).await?; let resp = crate::job_helpers_oss::load_preview_internal( - job_authed, &db, &w_id, file_key, inner, false, + job_authed, &db, None, &w_id, file_key, inner, false, ) .await?; Ok(Json(resp).into_response()) @@ -6346,6 +6705,127 @@ mod embed_token_tests { } } +#[cfg(test)] +mod operator_app_tests { + use super::{check_operator_composed_app, BuilderAppWrite, Policy}; + use windmill_common::error::Result; + use windmill_common::worker::to_raw_value; + + fn builder_policy(triggerables_v2: serde_json::Value) -> Policy { + serde_json::from_value(serde_json::json!({ + "execution_mode": "publisher", + "triggerables_v2": triggerables_v2, + })) + .unwrap() + } + + fn path_runnable(path: &str, run_type: &str) -> serde_json::Value { + serde_json::json!({"files": {}, "runnables": { + "a": {"name": "a", "type": "runnableByPath", "path": path, "runType": run_type} + }}) + } + + fn composed_app(value: serde_json::Value, policy: &mut Policy) -> Result> { + let value = to_raw_value(&value); + check_operator_composed_app(BuilderAppWrite::Deploy, true, Some(&value), Some(policy)) + } + + #[test] + fn operator_app_check_forces_the_sandbox_and_refuses_code() { + let clean = serde_json::json!({"files": {}, "runnables": { + "a": {"name": "a", "type": "runnableByPath", "path": "f/x/s", "runType": "script"} + }}); + + // A composition-only app goes through, sandboxed whether or not it asked to be. + let mut policy = builder_policy( + serde_json::json!({"a:script/f/x/s": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + let referenced = composed_app(clean.clone(), &mut policy).unwrap(); + assert_eq!(policy.sandbox, Some(true)); + // The runnables the policy authorizes are handed back for the caller to authorize. + assert_eq!(referenced, vec![(false, "f/x/s".to_string())]); + + // A hub reference is unreviewed code, refused like it is in a flow. + let mut policy = builder_policy( + serde_json::json!({"a:script/hub/1/x": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + + let mut policy = builder_policy(serde_json::json!({})); + policy.sandbox = Some(false); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + + // An inline script is refused even with no `language`, which the locking traversal skips. + let mut policy = builder_policy(serde_json::json!({})); + assert!(composed_app( + serde_json::json!({"runnables": {"a": {"inlineScript": {"content": "x"}}}}), + &mut policy + ) + .is_err()); + + // A `rawscript/` triggerable is what authorizes caller-supplied `raw_code` on the + // deployed app, so pinning one would be arbitrary code execution behind a clean value. + for key in ["rawscript/abc", "a:rawscript/abc"] { + let mut policy = builder_policy( + serde_json::json!({ key: {"static_inputs": {}, "one_of_inputs": {}} }), + ); + assert!( + composed_app(clean.clone(), &mut policy).is_err(), + "{key} must be refused" + ); + } + + // What the deployed bundle actually asks to run comes from the value's `runnableByPath` + // entries, which are a separate surface from the policy: both are reported. + let mut policy = builder_policy(serde_json::json!({})); + assert_eq!( + composed_app(path_runnable("f/x/s", "script"), &mut policy).unwrap(), + vec![(false, "f/x/s".to_string())] + ); + let mut policy = builder_policy(serde_json::json!({})); + assert_eq!( + composed_app(path_runnable("f/x/f", "flow"), &mut policy).unwrap(), + vec![(true, "f/x/f".to_string())] + ); + let mut policy = builder_policy(serde_json::json!({})); + assert!(composed_app(path_runnable("hub/1/x", "hubscript"), &mut policy).is_err()); + + // `execute_component` looks up `:` with an unrestricted component, so a + // second colon must not hide the path from validation: `x:y:script/hub/…` resolves at run + // time for `component = "x:y"`. + let mut policy = builder_policy( + serde_json::json!({"x:y:script/hub/123/foo": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + let mut policy = builder_policy( + serde_json::json!({"x:y:script/f/x/s": {"static_inputs": {}, "one_of_inputs": {}}}), + ); + assert_eq!( + composed_app(clean.clone(), &mut policy).unwrap(), + vec![(false, "f/x/s".to_string())] + ); + + // `Viewer` mode makes `execute_component` accept any script/flow path, triggerables or + // not, and run it as the viewer, so the checks above would stop binding. + let mut policy: Policy = serde_json::from_value(serde_json::json!({ + "execution_mode": "viewer", "triggerables_v2": {} + })) + .unwrap(); + assert!(composed_app(clean.clone(), &mut policy).is_err()); + + // Low-code apps stay closed. + let value = to_raw_value(&clean); + let mut policy = builder_policy(serde_json::json!({})); + assert!(check_operator_composed_app( + BuilderAppWrite::Deploy, + false, + Some(&value), + Some(&mut policy) + ) + .is_err()); + } +} + #[cfg(test)] mod policy_tests { /// `execution_mode` is optional in the OpenAPI schema the API clients and the @@ -6368,6 +6848,39 @@ mod policy_tests { } } +#[cfg(test)] +mod app_secret_tests { + use super::{id_from_secret, BUNDLE_SECRET_PREFIX}; + use magic_crypt::MagicCryptTrait; + + #[test] + fn every_invalid_secret_is_rejected_alike() { + let mc = magic_crypt::new_magic_crypt!("workspace-key", 256); + let other = magic_crypt::new_magic_crypt!("another-key", 256); + let valid = hex::encode(mc.encrypt_str_to_bytes("42")); + assert_eq!(id_from_secret(&mc, &valid, None), Some(42)); + let bundle = hex::encode(mc.encrypt_str_to_bytes(format!("{BUNDLE_SECRET_PREFIX}7"))); + assert_eq!( + id_from_secret(&mc, &bundle, Some(BUNDLE_SECRET_PREFIX)), + Some(7) + ); + + let mut tampered = mc.encrypt_str_to_bytes("42"); + *tampered.last_mut().unwrap() ^= 1; + for bad in [ + "not hex".to_string(), + hex::encode(&tampered), + hex::encode(other.encrypt_str_to_bytes("42")), + hex::encode(mc.encrypt_bytes_to_bytes(&[0xff, 0xfe])), + hex::encode(mc.encrypt_str_to_bytes("not a number")), + valid.clone(), + ] { + let prefix = (bad == valid).then_some(BUNDLE_SECRET_PREFIX); + assert_eq!(id_from_secret(&mc, &bad, prefix), None, "{bad}"); + } + } +} + #[cfg(test)] mod raw_app_file_key_tests { use super::validate_raw_app_path_keys; diff --git a/backend/windmill-api/src/drafts.rs b/backend/windmill-api/src/drafts.rs index a3cd34f3f8..e6b6db6919 100644 --- a/backend/windmill-api/src/drafts.rs +++ b/backend/windmill-api/src/drafts.rs @@ -15,15 +15,16 @@ use axum::{ }; use serde::{Deserialize, Serialize}; use windmill_api_flows::flows::validate_operator_flow; +use windmill_audit::{audit_oss::audit_log, ActionKind}; use windmill_common::{ db::UserDB, error::{Error, Result}, flows::FlowValue, user_drafts::{DraftUserRef, UserDraftItemKind, ENCRYPTED_DRAFT_PREFIX}, users::resolve_username_to_email, - utils::{check_proper_path, strip_json_nul}, + utils::{check_proper_path, require_admin, strip_json_nul}, variables::{build_crypt, encrypt}, - workspaces::operator_can_build_flows, + workspaces::{operator_builder_rights, BuilderKind}, }; pub fn workspaced_service() -> Router { @@ -32,6 +33,10 @@ pub fn workspaced_service() -> Router { .route("/get/{kind}/{*path}", get(get_draft_for_user)) .route("/get_own/{kind}/{*path}", get(get_own_draft)) .route("/update/{kind}/{*path}", post(update_draft)) + .route( + "/delete_for_user/{kind}/{*path}", + post(delete_draft_for_user), + ) .route("/move/{kind}/{*path}", post(move_draft)) .route("/migrate_legacy/{kind}/{*path}", post(migrate_legacy_draft)) } @@ -107,7 +112,7 @@ async fn list_drafts( // Without builder rights an operator has no drafts of their own (they can't write any, see // `require_can_write_path`), so this list is always empty for them. They can still READ some // collaborators' drafts via `/drafts/get`. - if authed.is_operator && !operator_can_build_flows(&db, &w_id).await? { + if authed.is_operator && !operator_builder_rights(&db, &w_id).await?.any() { return Ok(Json(vec![])); } let all_users = query.all_users.unwrap_or(false); @@ -496,6 +501,31 @@ async fn update_draft( } } + // A raw-app draft carries the app value and its policy side by side. + if authed.is_operator && kind == UserDraftItemKind::RawApp { + if let Some(value) = &req.value { + let draft: serde_json::Value = serde_json::from_str(&strip_json_nul(value.0.get())) + .map_err(|e| Error::BadRequest(format!("Invalid app draft: {e}")))?; + let mut policy = draft + .get("policy") + .filter(|p| !p.is_null()) + .map(|p| serde_json::from_value::(p.clone())) + .transpose() + .map_err(|e| Error::BadRequest(format!("Invalid app draft policy: {e}")))?; + crate::apps::validate_operator_composed_app( + &authed, + &db, + &user_db, + &w_id, + crate::apps::BuilderAppWrite::Draft, + true, + Some(&value.0), + policy.as_mut(), + ) + .await?; + } + } + let applied = if let Some(value) = &req.value { // Secret variable values must never sit in `draft.value` in plaintext // (see `encrypt_secret_variable_value`). @@ -1000,6 +1030,85 @@ async fn migrate_legacy_draft( } } +#[derive(Deserialize, Debug)] +pub struct DeleteDraftForUserQuery { + /// Workspace username of the draft owner. Absent: every draft at the path, + /// the legacy ownerless one and the caller's own included. + pub username: Option, +} + +/// Delete ANOTHER user's draft at a path, or every draft there, for workspace +/// admins (and superadmins, which carry `is_admin` in a workspace). Covers the +/// kinds whose authors are visible to others (`shares_drafts_across_users`); the +/// owner is named by workspace username, as on `/drafts/get`. +async fn delete_draft_for_user( + authed: ApiAuthed, + Extension(db): Extension, + Path((w_id, kind, path)): Path<(String, UserDraftItemKind, windmill_common::utils::StripPath)>, + Query(query): Query, +) -> Result { + require_admin(authed.is_admin, &authed.username)?; + if !kind.shares_drafts_across_users() { + return Err(Error::NotFound( + "drafts for this item kind are private to their owner".to_string(), + )); + } + let path = path.to_path(); + let username = query.username.as_deref(); + let owner_email = match username { + Some(username) => Some( + resolve_username_to_email(&w_id, username, &db) + .await? + .ok_or_else(|| { + Error::NotFound(format!("no user with username {username} in workspace")) + })?, + ), + None => None, + }; + + let mut tx = db.begin().await?; + let deleted = sqlx::query_scalar!( + r#"DELETE FROM draft + WHERE workspace_id = $1 AND path = $2 AND typ = $3 + AND ($4::text IS NULL OR email = $4) + RETURNING 1 as "one!""#, + &w_id, + path, + kind as UserDraftItemKind, + owner_email.as_deref(), + ) + .fetch_all(&mut *tx) + .await? + .len(); + if deleted == 0 { + return Err(Error::NotFound(match username { + Some(username) => format!("no draft for {username} at {path}"), + None => format!("no draft at {path}"), + })); + } + audit_log( + &mut *tx, + &authed, + "drafts.delete_for_user", + ActionKind::Delete, + &w_id, + Some(path), + Some( + [ + ("kind", kind.as_str()), + ("username", username.unwrap_or("*")), + ] + .into(), + ), + ) + .await?; + tx.commit().await?; + Ok(match username { + Some(username) => format!("Deleted {username}'s draft at {path}"), + None => format!("Deleted {deleted} draft(s) at {path}"), + }) +} + /// For variable-kind drafts with `variable.is_secret == true`, encrypt /// `variable.value` with the workspace crypt key and mark it /// `$encrypted:` so the secret never persists in plaintext at rest. @@ -1161,14 +1270,21 @@ pub(crate) async fn require_can_write_path( if authed.is_admin { return Ok(()); } - // Operators are read-only and never WRITE drafts, except a flow draft where the workspace - // granted the builder right: the kind has to be checked, or the right would open drafts of - // kinds it says nothing about. Read access is deliberately asymmetric: + // Operators are read-only and never WRITE drafts, except of a kind the workspace granted them + // the matching builder right for: a flows-only workspace must not get raw-app drafts through + // here. Read access is deliberately asymmetric: // `require_can_read_path` has no operator block, so an operator can still READ a draft they // can read via `/drafts/get`, mirroring their read access to deployed content. Intended. if authed.is_operator { - let granted = - matches!(kind, UserDraftItemKind::Flow) && operator_can_build_flows(db, w_id).await?; + let allowed = match kind { + UserDraftItemKind::Flow => Some(BuilderKind::Flows), + UserDraftItemKind::RawApp => Some(BuilderKind::Apps), + _ => None, + }; + let granted = match allowed { + Some(kind) => operator_builder_rights(db, w_id).await?.has(kind), + None => false, + }; if !granted { return Err(Error::PermissionDenied( "operators cannot save drafts".to_string(), diff --git a/backend/windmill-api/src/job_helpers_oss.rs b/backend/windmill-api/src/job_helpers_oss.rs index eb8986a986..0d6f20cdfa 100644 --- a/backend/windmill-api/src/job_helpers_oss.rs +++ b/backend/windmill-api/src/job_helpers_oss.rs @@ -263,6 +263,10 @@ mod app_s3_display_stubs { pub search_col: Option, pub search_term: Option, pub storage: Option, + pub s3_resource_path: Option, + pub csv_separator: Option, + pub csv_has_header: Option, + pub filters: Option, } #[derive(Serialize)] @@ -278,7 +282,10 @@ mod app_s3_display_stubs { pub search_col: Option, pub search_term: Option, pub storage: Option, + pub s3_resource_path: Option, pub csv_separator: Option, + pub csv_has_header: Option, + pub filters: Option, } pub async fn load_file_metadata_internal( @@ -308,6 +315,7 @@ mod app_s3_display_stubs { pub async fn load_table_count_internal( _authed: OptJobAuthed, _db: &DB, + _user_db: Option, _w_id: &str, _file_key: String, _query: LoadCountQuery, @@ -320,6 +328,7 @@ mod app_s3_display_stubs { pub async fn load_preview_internal( _authed: OptJobAuthed, _db: &DB, + _user_db: Option, _w_id: &str, _file_key: String, _query: LoadPreviewQuery, diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 5b0da21c50..29268d3cb2 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -55,7 +55,8 @@ use windmill_common::workspace_dependencies::{ RawWorkspaceDependencies, MIN_VERSION_WORKSPACE_DEPENDENCIES, }; use windmill_common::workspaces::{ - check_operator_can_build_flows, check_user_against_rule, ProtectionRuleKind, RuleCheckResult, + check_operator_can_build, check_user_against_rule, BuilderKind, ProtectionRuleKind, + RuleCheckResult, }; use windmill_common::DYNAMIC_INPUT_CACHE; #[cfg(all(feature = "enterprise", feature = "instance_smtp"))] @@ -2010,9 +2011,11 @@ async fn require_job_within_read_scope( } } // `scope_kind` is the runnable kind a `jobs:run::` scope can name, or - // NULL for a job no such scope reaches directly (previews, dependency jobs, - // flow-inlined scripts) — those are still readable as a step of a matching flow, - // through their ancestors. A `singlestepflow` wraps either a script or a flow, so it + // NULL for a job no such scope reaches directly (previews, flow-inlined scripts) — + // those are still readable as a step of a matching flow, through their ancestors. + // A dependency job's path is the one its `jobs:run:dependencies:` check + // admitted, but it is filed under `dependencies`, so it never passes for a run of + // the deployed runnable at that path. A `singlestepflow` wraps either a script or a flow, so it // projects onto the wrapped runnable the same way the batch-rerun query does. An agent // run is a preview of the one-step flow `agent_runs::agent_step_flow` builds, filed under the // agent's path (`.chat` for a chat turn); the editor's own runs of it look the same, @@ -2031,6 +2034,7 @@ async fn require_job_within_read_scope( WHEN a.agent THEN 'agents' WHEN j.kind IN ('script', 'script_hub', 'unassigned_script') THEN 'scripts' WHEN j.kind IN ('flow', 'unassigned_flow') THEN 'flows' + WHEN j.kind IN ('dependencies', 'flowdependencies') THEN 'dependencies' WHEN j.kind IN ('singlestepflow', 'unassigned_singlestepflow') THEN CASE WHEN COALESCE( (SELECT m->'value'->>'type' @@ -4628,14 +4632,8 @@ async fn cancel_selection( } else { ids }; - let Json(mut w_cancelled) = cancel_jobs( - ids, - &db, - &authed, - workspace_id.as_str(), - force_cancel, - ) - .await?; + let Json(mut w_cancelled) = + cancel_jobs(ids, &db, &authed, workspace_id.as_str(), force_cancel).await?; cancelled.append(&mut w_cancelled); } @@ -7826,6 +7824,7 @@ pub async fn run_workflow_as_code( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, mut tx) = push( &db, @@ -7854,7 +7853,7 @@ pub async fn run_workflow_as_code( None, push_authed.as_ref(), false, - None, + end_user_email, None, None, scope_ceiling.as_deref(), @@ -8141,6 +8140,7 @@ pub async fn run_wait_result_job_by_path_get( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, @@ -8169,7 +8169,7 @@ pub async fn run_wait_result_job_by_path_get( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(None), run_query.suspended_mode, scope_ceiling.as_deref(), @@ -8291,6 +8291,7 @@ pub async fn run_wait_result_script_by_path_internal( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, @@ -8319,7 +8320,7 @@ pub async fn run_wait_result_script_by_path_internal( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(None), run_query.suspended_mode, scope_ceiling.as_deref(), @@ -8409,6 +8410,7 @@ pub async fn run_wait_result_script_by_hash( ) }; + let end_user_email = run_end_user_email(&db, &w_id, &authed, None).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, @@ -8451,7 +8453,7 @@ pub async fn run_wait_result_script_by_hash( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(None), run_query.suspended_mode, scope_ceiling.as_deref(), @@ -9525,13 +9527,23 @@ pub struct RunDependenciesResponse { pub dependencies: String, } +/// The path comes from the request body: one carrying scope syntax (`,`, `:`, `*`) could +/// put a granted path beside an ungranted one, so it needs a grant covering every path. +fn check_dependencies_scope(authed: &ApiAuthed, path: &str) -> error::Result<()> { + if windmill_common::auth::is_scope_literal_path(path) { + check_scopes(authed, || format!("jobs:run:dependencies:{path}")) + } else { + check_scopes(authed, || "jobs:run:dependencies".to_string()) + } +} + async fn push_dependencies_job( authed: &ApiAuthed, db: &DB, w_id: &str, req: RunDependenciesRequest, ) -> error::Result { - check_scopes(authed, || format!("jobs:run"))?; + check_dependencies_scope(authed, &req.entrypoint)?; if authed.is_operator { return Err(error::Error::NotAuthorized( "Operators cannot run dependencies jobs for security reasons".to_string(), @@ -9654,8 +9666,15 @@ async fn push_flow_dependencies_job( w_id: &str, req: RunFlowDependenciesRequest, ) -> error::Result { - check_scopes(authed, || format!("jobs:run"))?; - check_operator_can_build_flows(db, w_id, authed.is_operator, "run dependencies jobs").await?; + check_dependencies_scope(authed, &req.path)?; + check_operator_can_build( + db, + w_id, + authed.is_operator, + BuilderKind::Flows, + "run dependencies jobs", + ) + .await?; // The dependency job locks whatever inline code this request carries, on a worker. A // composition-only flow has none, so validating here costs a builder nothing and keeps the // lock step from becoming the way to run code the write path refuses. @@ -10070,7 +10089,14 @@ async fn run_preview_flow_job( Query(run_query): Query, Json(raw_flow): Json, ) -> error::Result<(StatusCode, String)> { - check_operator_can_build_flows(&db, &w_id, authed.is_operator, "run preview jobs").await?; + check_operator_can_build( + &db, + &w_id, + authed.is_operator, + BuilderKind::Flows, + "run preview jobs", + ) + .await?; // Flow preview runs an arbitrary, request-supplied flow definition; require the broad // jobs:run scope so a narrowly-scoped token cannot escape its scope. See run_preview_script. check_scopes(&authed, || format!("jobs:run"))?; @@ -10556,6 +10582,7 @@ pub async fn run_job_by_hash_inner( }; let job_payload = with_run_retry(&run_query, job_payload, &push_args, &tag)?; + let end_user_email = run_end_user_email(&db, &w_id, &authed, trigger.as_ref()).await?; let scope_ceiling = windmill_api_auth::caller_scope_ceiling(&db, &authed).await?; let (uuid, tx) = push( &db, @@ -10584,7 +10611,7 @@ pub async fn run_job_by_hash_inner( None, push_authed.as_ref(), false, - None, + end_user_email, authed.trigger_or_fallback(trigger), run_query.suspended_mode, scope_ceiling.as_deref(), diff --git a/backend/windmill-api/src/live_migrations.rs b/backend/windmill-api/src/live_migrations.rs index 334ee340a8..a620807902 100644 --- a/backend/windmill-api/src/live_migrations.rs +++ b/backend/windmill-api/src/live_migrations.rs @@ -8,7 +8,9 @@ use sqlx::{PgConnection, Postgres}; use tokio::task::JoinHandle; -use windmill_common::{db::DB, error::Error}; +use windmill_common::{ + db::DB, error::Error, global_settings::SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING, +}; use crate::db::CustomMigrator; use sqlx::migrate::Migrate; @@ -160,7 +162,11 @@ async fn run_background_migrations(db: &DB) -> Result<(), Error> { return Ok(()); } - for step in [AUDIT_OPERATION_INDEX, RETIRE_LEGACY_AUDIT] { + for step in [ + RETIRE_LEGACY_HUB_SYNC, + AUDIT_OPERATION_INDEX, + RETIRE_LEGACY_AUDIT, + ] { if background_migration_done(&mut conn, step).await? { continue; } @@ -171,6 +177,7 @@ async fn run_background_migrations(db: &DB) -> Result<(), Error> { let run = match step { AUDIT_OPERATION_INDEX => create_audit_operation_index(&mut conn).await, RETIRE_LEGACY_AUDIT => retire_legacy_audit_table(&mut conn).await, + RETIRE_LEGACY_HUB_SYNC => retire_legacy_hub_sync(&mut conn).await, _ => unreachable!("background migration {step} has no step function"), }; match run { @@ -192,6 +199,7 @@ async fn run_background_migrations(db: &DB) -> Result<(), Error> { const AUDIT_OPERATION_INDEX: &str = "audit_partitioned_workspace_operation_index"; const RETIRE_LEGACY_AUDIT: &str = "retire_legacy_audit_table"; +const RETIRE_LEGACY_HUB_SYNC: &str = "retire_legacy_hub_sync_script"; // Short on purpose: a statement waiting for its lock is also a wait for every audit insert // queued behind it. @@ -371,6 +379,70 @@ async fn retire_legacy_audit_table(conn: &mut PgConnection) -> Result<(), Error> Ok(()) } +/// The hash the migrations gave `u/admin/hub_sync`; they rewrote its content in place. +const LEGACY_HUB_SYNC_HASH: i64 = -28028598712388162; + +/// `u/admin/hub_sync` and the schedules running it synced resource types before servers did +/// it themselves (`windmill_common::hub_resource_types`). Retired only while the script is +/// still the version Windmill created: a copy someone edited is theirs to keep running. +async fn retire_legacy_hub_sync(conn: &mut PgConnection) -> Result<(), Error> { + let mut tx = conn.begin().await?; + let latest: Option = sqlx::query_scalar( + "SELECT hash FROM script + WHERE workspace_id = 'admins' AND path = 'u/admin/hub_sync' AND archived = false + ORDER BY created_at DESC LIMIT 1", + ) + .fetch_optional(&mut *tx) + .await?; + if latest != Some(LEGACY_HUB_SYNC_HASH) { + return Ok(()); + } + + let schedules: Vec<(String, bool, serde_json::Value)> = sqlx::query_as( + "SELECT path, enabled, jsonb_build_object('row', to_jsonb(s)) FROM schedule s + WHERE workspace_id = 'admins' AND script_path = 'u/admin/hub_sync' AND NOT is_flow", + ) + .fetch_all(&mut *tx) + .await?; + // An enabled schedule hands over to the server's daily sync in the transaction that deletes + // it, so the instance never has both syncs running, nor neither. + if schedules.iter().any(|(_, enabled, _)| *enabled) { + let inserted = sqlx::query( + "INSERT INTO global_settings (name, value) VALUES ($1, 'true'::jsonb) + ON CONFLICT (name) DO NOTHING", + ) + .bind(SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING) + .execute(&mut *tx) + .await? + .rows_affected(); + if inserted > 0 { + tracing::info!( + "u/admin/hub_sync ran on an enabled schedule: turned on {SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING}" + ); + } + } + for (path, _, row) in schedules { + windmill_queue::schedule::clear_schedule(&mut tx, &path, "admins").await?; + sqlx::query("DELETE FROM schedule WHERE workspace_id = 'admins' AND path = $1") + .bind(&path) + .execute(&mut *tx) + .await?; + windmill_common::trashbin::move_to_trash( + &mut *tx, "admins", "schedule", &path, row, "system", + ) + .await?; + tracing::info!("Deleted schedule {path}, which ran u/admin/hub_sync"); + } + sqlx::query( + "UPDATE script SET archived = true WHERE workspace_id = 'admins' AND path = 'u/admin/hub_sync'", + ) + .execute(&mut *tx) + .await?; + tx.commit().await?; + tracing::info!("Archived u/admin/hub_sync: servers sync resource types themselves"); + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -412,4 +484,80 @@ mod tests { assert_eq!(seen, 2); Ok(()) } + + async fn insert_legacy_hub_sync_schedule(db: &DB, enabled: bool) -> anyhow::Result<()> { + sqlx::query( + "INSERT INTO schedule (workspace_id, path, edited_by, email, permissioned_as, schedule, timezone, script_path, is_flow, enabled) + VALUES ('admins', 'g/all/hub_sync', 'admin', 'admin@windmill.dev', 'u/admin', '0 0 0 * * *', 'Etc/UTC', 'u/admin/hub_sync', false, $1)", + ) + .bind(enabled) + .execute(db) + .await?; + Ok(()) + } + + async fn schedules_and_live_versions(db: &DB) -> anyhow::Result<(i64, i64)> { + Ok(sqlx::query_as( + "SELECT (SELECT count(*) FROM schedule WHERE workspace_id = 'admins'), + (SELECT count(*) FROM script WHERE path = 'u/admin/hub_sync' AND NOT archived)", + ) + .fetch_one(db) + .await?) + } + + async fn daily_sync_setting(db: &DB) -> anyhow::Result> { + Ok( + sqlx::query_scalar("SELECT value FROM global_settings WHERE name = $1") + .bind(SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING) + .fetch_optional(db) + .await?, + ) + } + + #[sqlx::test(migrations = "../migrations")] + async fn retire_legacy_hub_sync_spares_an_edited_script(db: DB) -> anyhow::Result<()> { + insert_legacy_hub_sync_schedule(&db, true).await?; + sqlx::query( + "INSERT INTO script (workspace_id, hash, path, content, created_by, language, summary, description, schema, lock, created_at) + SELECT workspace_id, 42, path, content || '// edited', 'admin', language, summary, description, schema, lock, now() + interval '1 minute' + FROM script WHERE workspace_id = 'admins' AND path = 'u/admin/hub_sync'", + ) + .execute(&db) + .await?; + + retire_legacy_hub_sync(&mut *db.acquire().await?).await?; + assert_eq!(schedules_and_live_versions(&db).await?, (1, 2)); + assert_eq!(daily_sync_setting(&db).await?, None); + + // Unedited, the enabled schedule hands over to the server's daily sync. + sqlx::query("DELETE FROM script WHERE hash = 42") + .execute(&db) + .await?; + retire_legacy_hub_sync(&mut *db.acquire().await?).await?; + assert_eq!(schedules_and_live_versions(&db).await?, (0, 0)); + assert_eq!( + daily_sync_setting(&db).await?, + Some(serde_json::json!(true)) + ); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations")] + async fn retire_legacy_hub_sync_leaves_the_daily_sync_off_without_an_enabled_schedule( + db: DB, + ) -> anyhow::Result<()> { + for disabled_schedule in [true, false] { + sqlx::query("UPDATE script SET archived = false WHERE path = 'u/admin/hub_sync'") + .execute(&db) + .await?; + if disabled_schedule { + insert_legacy_hub_sync_schedule(&db, false).await?; + } + + retire_legacy_hub_sync(&mut *db.acquire().await?).await?; + assert_eq!(schedules_and_live_versions(&db).await?, (0, 0)); + assert_eq!(daily_sync_setting(&db).await?, None); + } + Ok(()) + } } diff --git a/backend/windmill-api/src/runnables.rs b/backend/windmill-api/src/runnables.rs index 846089630b..7e64b48a1a 100644 --- a/backend/windmill-api/src/runnables.rs +++ b/backend/windmill-api/src/runnables.rs @@ -34,7 +34,7 @@ use std::collections::HashMap; use windmill_common::{ db::UserDB, error::{Error, JsonResult}, - workspaces::operator_can_build_flows, + workspaces::{operator_builder_rights, OperatorBuilderRights}, }; use windmill_types::scripts::ScriptHash; use windmill_types::user_drafts::DraftUserRef; @@ -574,9 +574,14 @@ async fn list_runnables( // Draft-only rows are the caller's own work in progress: never archived, so they // have no place in the archived view, and carrying no labels of their own they are // out of scope of a label filter (as in the per-kind endpoints). Operators don't - // see other people's drafts and have none of their own, except a builder's flows. + // see other people's drafts and have none of their own, except of what they may build. + let builder = if authed.is_operator { + operator_builder_rights(&db, &w_id).await? + } else { + OperatorBuilderRights::default() + }; let include_drafts = q.include_draft_only.unwrap_or(false) - && (!authed.is_operator || operator_can_build_flows(&db, &w_id).await?) + && (!authed.is_operator || builder.any()) && !show_archived && q.label.as_ref().filter(|s| !s.is_empty()).is_none(); let draft_extras_for = |kind: &str| -> Vec { @@ -662,7 +667,10 @@ async fn list_runnables( keyset: Option<&str>, limit: Option| -> Option { - if !include_drafts || !kinds.contains(&kind) || (authed.is_operator && kind != "flow") { + if !include_drafts + || !kinds.contains(&kind) + || (authed.is_operator && !operator_drafts_kind(builder, kind)) + { return None; } // `fav` is ignored: with no favorite join there is nothing to filter on, and the @@ -990,6 +998,15 @@ async fn count_runnables_by_owner( Ok(Json(RunnableCountsResponse { counts })) } +/// Whether an operator with `builder` rights can have drafts of a `/list` kind. +fn operator_drafts_kind(builder: OperatorBuilderRights, kind: &str) -> bool { + match kind { + "flow" => builder.flows, + "app" => builder.apps, + _ => false, + } +} + /// Adds the caller's draft-only rows to `counts`, in the same shape `/list` /// returns them so a badge never disagrees with the rows behind it. /// @@ -1010,13 +1027,21 @@ async fn add_draft_counts( if !q.include_draft_only.unwrap_or(false) { return Ok(()); } - // An operator has no drafts of their own, except a builder's flows. + // An operator has no drafts of their own, except of what they may build. + let builder_kinds: Vec<&str>; let kinds: &[&str] = if !authed.is_operator { kinds - } else if kinds.contains(&"flow") && operator_can_build_flows(db, w_id).await? { - &["flow"] } else { - return Ok(()); + let builder = operator_builder_rights(db, w_id).await?; + builder_kinds = kinds + .iter() + .copied() + .filter(|k| operator_drafts_kind(builder, k)) + .collect(); + if builder_kinds.is_empty() { + return Ok(()); + } + &builder_kinds }; // $1 = workspace, $2 = the caller's email. let mut binds: Vec = vec![]; diff --git a/backend/windmill-api/src/token.rs b/backend/windmill-api/src/token.rs index b7fbe96404..ab22f2d15b 100644 --- a/backend/windmill-api/src/token.rs +++ b/backend/windmill-api/src/token.rs @@ -215,6 +215,11 @@ lazy_static! { label: "Run flows".to_string(), requires_resource_path: true, }, + ScopeOption { + value: "jobs:run:dependencies".to_string(), + label: "Generate dependency locks".to_string(), + requires_resource_path: true, + }, ], }]; diff --git a/backend/windmill-common/src/apps.rs b/backend/windmill-common/src/apps.rs index 36b2d9b704..1686c20d2b 100644 --- a/backend/windmill-common/src/apps.rs +++ b/backend/windmill-common/src/apps.rs @@ -18,6 +18,59 @@ lazy_static::lazy_static! { pub static ref APP_WORKSPACED_ROUTE: AtomicBool = AtomicBool::new(false); } +/// Whether the app value carries an `inlineScript` anywhere. +/// +/// Deliberately not built on [`traverse_app_inline_scripts`], which only reports a script whose +/// `language` parses and stops descending as soon as it sees an `inlineScript` key. That is right +/// for locking (nothing to lock without a language) and wrong for an authorization check, where +/// the author picks the fields: this refuses the key itself, whatever it contains. +pub fn app_value_has_inline_script(value: &Value) -> bool { + match value { + Value::Object(object) => { + object.contains_key("inlineScript") || object.values().any(app_value_has_inline_script) + } + Value::Array(array) => array.iter().any(app_value_has_inline_script), + _ => false, + } +} + +/// Every workspace runnable the app value points a component at, as `(is_flow, path)`. +/// +/// This is what the deployed bundle actually asks `execute_component` to run: it resolves a +/// `runnable_id` against the stored `runnables` and sends `{runType}/{path}`. The policy's +/// triggerables are a separate surface, so both have to be authorized. Anything not plain text +/// is refused, never skipped: the editor compares `type` loosely (`["path"] == "path"`) and +/// interpolates `path` (`["u/x"]` → `u/x`), so a skipped entry still runs. +pub fn app_value_runnable_paths(value: &Value) -> error::Result> { + let bad = |msg: String| error::Error::BadRequest(msg); + let runnables = match value.get("runnables") { + None | Some(Value::Null) => return Ok(vec![]), + Some(Value::Object(runnables)) => runnables, + Some(_) => return Err(bad("App runnables must be an object".to_string())), + }; + let mut out = Vec::new(); + for (id, runnable) in runnables { + let Value::Object(runnable) = runnable else { + return Err(bad(format!("Runnable {id} must be an object"))); + }; + match runnable.get("type") { + Some(Value::String(t)) if t == "runnableByPath" || t == "path" => {} + None | Some(Value::String(_)) => continue, + Some(_) => return Err(bad(format!("Runnable {id} has a non-text type"))), + } + let Some(Value::String(path)) = runnable.get("path") else { + return Err(bad(format!("Runnable {id} must name its path as text"))); + }; + let is_flow = match runnable.get("runType") { + Some(Value::String(t)) if t == "flow" => true, + Some(Value::String(t)) if t == "script" || t == "hubscript" => false, + _ => return Err(bad(format!("Unsupported runType for runnable {path}"))), + }; + out.push((is_flow, path.clone())); + } + Ok(out) +} + /// Traverse FlowValue while invoking provided by caller callback on leafs // #[async_recursion::async_recursion(?Send)] pub fn traverse_app_inline_scripts< diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index e1a731f3e6..2bc303f5c2 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -158,6 +158,7 @@ pub const SSO_GROUPS_CLAIM_SETTING: &str = "sso_groups_claim"; pub const HUB_BASE_URL_SETTING: &str = "hub_base_url"; pub const HUB_ACCESSIBLE_URL_SETTING: &str = "hub_accessible_url"; pub const DISABLE_HUB_SETTING: &str = "disable_hub"; +pub const SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING: &str = "sync_hub_resource_types_daily"; pub const CRITICAL_ERROR_CHANNELS_SETTING: &str = "critical_error_channels"; pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const CRITICAL_ALERTS_ON_DB_OVERSIZE_SETTING: &str = "critical_alerts_on_db_oversize"; @@ -181,6 +182,12 @@ pub const GITHUB_ENTERPRISE_APP_SETTING: &str = "github_enterprise_app"; /// `base_url` when unset; set it when the browser-facing URL is not reachable /// from GitHub and a separate ingress fronts the API for inbound webhooks. pub const GITHUB_APP_WEBHOOK_BASE_URL_SETTING: &str = "github_app_webhook_base_url"; +/// Base URL the UI shows for the endpoints external clients call (webhooks, HTTP +/// routes, push trigger endpoints, the CLI), instead of the origin the UI is browsed on. +/// For deployments that serve the API on a separate domain or behind a gateway. +/// EE only. Readable by any authenticated user, like [`INSTANCE_BANNER_SETTING`]: +/// every user is shown URLs built from it. +pub const API_BASE_URL_SETTING: &str = "api_base_url"; /// Instance-wide announcement rendered above every page of the app (maintenance /// windows, incidents). Readable by any authenticated user, unlike most settings: /// the banner exists to be shown to everyone, so it must never hold anything the @@ -313,12 +320,17 @@ pub fn validate_accent_color(value: &serde_json::Value) -> Result<(), String> { pub struct InstanceUi { pub instance_banner: Option, pub accent_color: Option, + pub api_base_url: Option, } pub async fn get_instance_ui(db: &Pool) -> error::Result { let rows = sqlx::query!( "SELECT name, value FROM global_settings WHERE name = ANY($1)", - &[INSTANCE_BANNER_SETTING, ACCENT_COLOR_SETTING] as &[&str] + &[ + INSTANCE_BANNER_SETTING, + ACCENT_COLOR_SETTING, + API_BASE_URL_SETTING + ] as &[&str] ) .fetch_all(db) .await?; @@ -327,17 +339,22 @@ pub async fn get_instance_ui(db: &Pool) -> error::Result { match row.name.as_str() { INSTANCE_BANNER_SETTING => ui.instance_banner = Some(row.value), ACCENT_COLOR_SETTING => ui.accent_color = Some(row.value), + API_BASE_URL_SETTING => ui.api_base_url = Some(row.value), _ => {} } } Ok(ui) } -/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] value. +const SHELL_ACTIVE_CHARS: [char; 16] = [ + '$', '`', '\'', '"', '\\', ';', '&', '|', '<', '>', '(', ')', '{', '}', '*', '!', +]; + +/// Validate a [`GITHUB_APP_WEBHOOK_BASE_URL_SETTING`] or [`API_BASE_URL_SETTING`] value. /// -/// The receiver path is appended to it verbatim, so anything that doesn't -/// concatenate into a URL GitHub can POST to must be rejected at write time -/// rather than silently producing an unreachable hook: a wrong scheme +/// A path is appended to it verbatim, so anything that doesn't concatenate into a +/// URL an external caller can reach must be rejected at write time rather than +/// silently producing an unreachable hook or endpoint: a wrong scheme /// (`httpss://`), a missing host, embedded whitespace, or a query/fragment /// (appending a path after `?`/`#` keeps it inside the query/fragment). /// @@ -353,7 +370,7 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> { url::Url::parse(value).map_err(|e| format!("must be an absolute http(s) URL: {e}"))?; if !url.username().is_empty() || url.password().is_some() { return Err( - "must not embed a username or password: the receiver URL is stored in workspace settings, where it is readable by workspace admins".to_string(), + "must not embed a username or password: the URL is readable by users who are not instance admins".to_string(), ); } if !matches!(url.scheme(), "http" | "https") { @@ -366,13 +383,21 @@ pub fn validate_webhook_base_url(value: &str) -> Result<(), String> { } if url.query().is_some() || url.fragment().is_some() { return Err( - "must not include a query string or fragment, since the webhook path is appended to it" + "must not include a query string or fragment, since a path is appended to it" .to_string(), ); } if value.chars().any(char::is_whitespace) { return Err("must not contain whitespace".to_string()); } + // The UI pastes the URL into shell commands users copy, where double quotes + // do not stop `$(...)` or backticks from running. + if value.contains(SHELL_ACTIVE_CHARS) { + return Err( + "must not contain shell metacharacters, since it is shown inside commands users copy" + .to_string(), + ); + } // Matches the sibling `base_url` / `hub_base_url` convention. The receiver // builder trims it anyway, so this is about keeping the stored value canonical // rather than about reachability. @@ -1101,6 +1126,9 @@ mod tests { "https://hooks.example.com#", "https://user:password@hooks.example.com", "https://user@hooks.example.com", + "https://hooks.example.com/$(uname)", + "https://hooks.example.com/`uname`", + "https://hooks.example.com/a;b", ]; for v in accept { assert!( diff --git a/backend/windmill-common/src/hub_resource_types.rs b/backend/windmill-common/src/hub_resource_types.rs new file mode 100644 index 0000000000..88de520094 --- /dev/null +++ b/backend/windmill-common/src/hub_resource_types.rs @@ -0,0 +1,578 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +//! Keeps the `admins` workspace's resource types, which every workspace falls back to, in +//! step with the hub. Servers run [`spawn_daily_sync`] once a superadmin turns it on, +//! superadmins run [`sync`] on demand, and `windmill cache-rt` bakes the public hub's listing +//! into the image for instances that cannot reach a hub. + +use std::collections::HashMap; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use serde_json::json; + +use crate::{ + error::{Error, Result}, + global_settings::{ + load_value_from_global_settings, DISABLE_HUB_SETTING, SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING, + }, + utils::{get_license_id_or_uid, HTTP_CLIENT_PERMISSIVE, HUB_API_SECRET}, + worker::HUB_RT_CACHE_DIR, + DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, +}; + +const SYNC_INTERVAL: chrono::TimeDelta = chrono::TimeDelta::hours(24); +const RETRY_INTERVAL: chrono::TimeDelta = chrono::TimeDelta::hours(1); +const CHECK_INTERVAL: std::time::Duration = std::time::Duration::from_secs(600); +const SYNC_LOCK_ID: i64 = 737_483_925; +/// `background_task_state.name` of the sync's last attempt and last hub sync. +const SYNC_STATE_TASK: &str = "hub_resource_type_sync"; + +pub fn cache_path() -> String { + format!("{}/resource_types.json", *HUB_RT_CACHE_DIR) +} + +/// A hub resource type with its schema parsed, as synced and as `cache-rt` stores it. +/// +/// `format_extension` and `display_name` are doubly optional: no key (a cache written before +/// the field) leaves the stored value alone, while an explicit null clears it. Plain serde +/// folds both into `None`, hence the wrapping deserializer. +#[derive(Deserialize, Serialize, Clone)] +pub struct HubResourceType { + pub name: String, + pub schema: Option, + pub description: Option, + #[serde( + default, + deserialize_with = "crate::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub format_extension: Option>, + #[serde( + default, + deserialize_with = "crate::more_serde::double_option", + skip_serializing_if = "Option::is_none" + )] + pub display_name: Option>, + /// Absent from hubs that do not publish filesets, which leaves the stored flag alone. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub is_fileset: Option, +} + +/// An entry of the hub's `/resource_types/list`, whose schema is a JSON string. +#[derive(Deserialize)] +struct HubListing { + name: String, + schema: Option, + description: Option, + /// Absent from hubs predating the column, which do not publish file types either, so + /// the hub is authoritative for it whether or not it sends the key. + #[serde(default)] + format_extension: Option, + #[serde(default, deserialize_with = "crate::more_serde::double_option")] + display_name: Option>, + #[serde(default)] + is_fileset: Option, +} + +/// One hub's listing. A type whose schema does not parse is left out. +pub async fn fetch( + hub_base_url: &str, + api_secret: Option<&str>, + uid: Option<&str>, +) -> Result> { + // Permissive, so `ACCEPT_INVALID_CERTS` lets the sync reach a private hub behind a + // self-signed certificate. + let mut request = HTTP_CLIENT_PERMISSIVE + .get(format!("{hub_base_url}/resource_types/list")) + .header("Accept", "application/json"); + if let Some(uid) = uid { + request = request.header("X-uid", uid); + } + if let Some(secret) = api_secret { + request = request.header("X-api-secret", secret); + } + let response = request.send().await.map_err(|e| { + Error::InternalErr(format!("Failed to reach the hub at {hub_base_url}: {e}")) + })?; + let status = response.status(); + if status == reqwest::StatusCode::UNAUTHORIZED { + return Err(Error::InternalErr(format!( + "The hub at {hub_base_url} refused the request: check the Private Hub API secret" + ))); + } + if !status.is_success() { + return Err(Error::InternalErr(format!( + "The hub at {hub_base_url} answered {status}" + ))); + } + let listing: Vec = response.json().await.map_err(|e| { + Error::InternalErr(format!( + "Failed to parse the resource types listed by {hub_base_url}: {e}" + )) + })?; + + Ok(listing + .into_iter() + .filter_map(|rt| { + let schema = match rt.schema.as_deref().map(serde_json::from_str).transpose() { + Ok(schema) => schema, + Err(e) => { + tracing::warn!( + "Skipping hub resource type {}: invalid schema: {e}", + rt.name + ); + return None; + } + }; + Some(HubResourceType { + name: rt.name, + schema, + description: rt.description, + format_extension: Some(rt.format_extension), + display_name: rt.display_name, + is_fileset: rt.is_fileset, + }) + }) + .collect()) +} + +/// The listing of the hub this instance is configured with. +async fn fetch_configured_hub(db: &DB) -> Result> { + let hub = (**HUB_BASE_URL.load()).clone(); + let uid = get_license_id_or_uid(db).await.ok(); + if hub == DEFAULT_HUB_BASE_URL { + return fetch(&hub, None, uid.as_deref()).await; + } + let secret = (**HUB_API_SECRET.load()).clone(); + let types = fetch(&hub, secret.as_deref(), uid.as_deref()).await?; + if !types.is_empty() { + return Ok(types); + } + // A private hub that publishes no resource types of its own defers to the public one, + // which must not be sent the private hub's secret. + tracing::info!("The hub at {hub} lists no resource types, syncing the public hub's"); + fetch(DEFAULT_HUB_BASE_URL, None, uid.as_deref()).await +} + +#[derive(PartialEq, Debug)] +struct ResourceTypeRow { + schema: Option, + description: Option, + format_extension: Option, + is_fileset: bool, + display_name: Option, +} + +/// The row a synced type should end up as, given what `admins` already stores for it. +fn target_row(rt: &HubResourceType, stored: Option<&ResourceTypeRow>) -> ResourceTypeRow { + let is_fileset = rt + .is_fileset + .unwrap_or_else(|| stored.is_some_and(|s| s.is_fileset)); + // A fileset is a set of files, so it cannot also be one file. Create and update reject + // the pair; this writer bypasses both. + let format_extension = if is_fileset { + None + } else { + match &rt.format_extension { + Some(extension) => extension.clone(), + None => stored.and_then(|s| s.format_extension.clone()), + } + }; + // A name too long for the column counts as absent: one bad entry must not fail the + // upsert and end the rest of the sync. + let display_name = match &rt.display_name { + Some(Some(name)) if name.chars().count() > 100 => { + tracing::warn!( + "Ignoring the display_name of resource type {}: longer than 100 characters", + rt.name + ); + stored.and_then(|s| s.display_name.clone()) + } + Some(name) => name.clone(), + None => stored.and_then(|s| s.display_name.clone()), + }; + ResourceTypeRow { + schema: rt.schema.clone(), + description: rt.description.clone(), + format_extension, + is_fileset, + display_name, + } +} + +#[derive(Clone, Copy, PartialEq, Debug)] +pub enum SyncSource { + Hub, + ImageCache, +} + +pub struct SyncOutcome { + pub source: SyncSource, + /// Every type the source listed, changed or not. + pub listed: Vec, + /// The types this sync created or updated. + pub changed: Vec, +} + +impl SyncOutcome { + pub fn summary(&self) -> String { + let synced = self.changed.len(); + let unchanged = self.listed.len() - synced; + match self.source { + SyncSource::Hub => { + format!("Synced {synced} resource types from the hub ({unchanged} unchanged)") + } + SyncSource::ImageCache => format!( + "Synced {synced} resource types from the image's cache ({unchanged} unchanged)" + ), + } + } +} + +async fn apply(db: &DB, source: SyncSource, types: &[HubResourceType]) -> Result { + let stored: HashMap = sqlx::query_as::< + _, + ( + String, + Option, + Option, + Option, + bool, + Option, + ), + >( + "SELECT name, schema, description, format_extension, is_fileset, display_name + FROM resource_type WHERE workspace_id = 'admins'", + ) + .fetch_all(db) + .await? + .into_iter() + .map( + |(name, schema, description, format_extension, is_fileset, display_name)| { + ( + name, + ResourceTypeRow { schema, description, format_extension, is_fileset, display_name }, + ) + }, + ) + .collect(); + + let mut changed = vec![]; + for rt in types { + let current = stored.get(&rt.name); + let target = target_row(rt, current); + if current == Some(&target) { + continue; + } + sqlx::query( + "INSERT INTO resource_type + (workspace_id, name, schema, description, format_extension, is_fileset, display_name, edited_at) + VALUES ('admins', $1, $2, $3, $4, $5, $6, now()) + ON CONFLICT (workspace_id, name) DO UPDATE + SET schema = EXCLUDED.schema, description = EXCLUDED.description, + format_extension = EXCLUDED.format_extension, is_fileset = EXCLUDED.is_fileset, + display_name = EXCLUDED.display_name, edited_at = now()", + ) + .bind(&rt.name) + .bind(&target.schema) + .bind(&target.description) + .bind(&target.format_extension) + .bind(target.is_fileset) + .bind(&target.display_name) + .execute(db) + .await?; + changed.push(rt.name.clone()); + } + + Ok(SyncOutcome { source, listed: types.iter().map(|rt| rt.name.clone()).collect(), changed }) +} + +#[derive(Deserialize, Default)] +struct SyncState { + last_attempt_at: Option>, + last_hub_sync_at: Option>, +} + +async fn load_state(db: &DB) -> Result { + let value: Option = + sqlx::query_scalar("SELECT value FROM background_task_state WHERE name = $1") + .bind(SYNC_STATE_TASK) + .fetch_optional(db) + .await?; + Ok(value + .and_then(|v| serde_json::from_value(v).ok()) + .unwrap_or_default()) +} + +/// Merged into the stored state, so a writer only ever sets the keys it names. +async fn record_state(db: &DB, patch: serde_json::Value) -> Result<()> { + sqlx::query( + "INSERT INTO background_task_state (name, value) VALUES ($1, $2) + ON CONFLICT (name) DO UPDATE + SET value = background_task_state.value || EXCLUDED.value, updated_at = now()", + ) + .bind(SYNC_STATE_TASK) + .bind(patch) + .execute(db) + .await?; + Ok(()) +} + +async fn setting_enabled(db: &DB, name: &str) -> Result { + Ok(load_value_from_global_settings(db, name) + .await? + .and_then(|v| v.as_bool()) + .unwrap_or(false)) +} + +/// Syncs from the configured hub and records the attempt, successful or not. +async fn sync_from_hub(db: &DB) -> Result { + let attempted_at = Utc::now(); + let result = match fetch_configured_hub(db).await { + Ok(types) => apply(db, SyncSource::Hub, &types).await, + Err(e) => Err(e), + }; + let state = match &result { + Ok(_) => { + json!({ "last_attempt_at": attempted_at, "last_hub_sync_at": attempted_at, "last_error": null }) + } + Err(e) => json!({ "last_attempt_at": attempted_at, "last_error": e.to_string() }), + }; + record_state(db, state).await?; + result +} + +/// Syncs from the configured hub, falling back to the image's cache when the hub is disabled +/// or unreachable and the instance has never synced from a hub. +pub async fn sync(db: &DB) -> Result { + let hub_err = if setting_enabled(db, DISABLE_HUB_SETTING).await? { + Error::BadRequest("The hub is disabled on this instance".to_string()) + } else { + match sync_from_hub(db).await { + Ok(outcome) => return Ok(outcome), + Err(e) => e, + } + }; + match fall_back_to_image_cache(db, &cache_path()).await? { + Some(outcome) => { + tracing::warn!("Hub resource type sync failed, used the image's cache: {hub_err}"); + Ok(outcome) + } + None => Err(hub_err), + } +} + +/// The cache is the listing as of the image's build, so standing in for the hub on an +/// instance that has synced from one would revert every type the hub has updated since. +async fn fall_back_to_image_cache(db: &DB, path: &str) -> Result> { + if let Some(at) = load_state(db).await?.last_hub_sync_at { + tracing::info!( + "Not applying the image's cached resource types: synced from the hub at {at}" + ); + return Ok(None); + } + sync_from_cache_file(db, path).await +} + +/// Applies the listing baked into the image (`SYNC_CACHED_RT`), on every start and whether or +/// not the instance has synced from a hub: it is how an instance that cannot reach the public +/// hub, a private hub listing only its own types included, takes each new image's public types. +/// `None` when the image carries no cache. +pub async fn sync_from_image_cache(db: &DB) -> Result> { + sync_from_cache_file(db, &cache_path()).await +} + +async fn sync_from_cache_file(db: &DB, path: &str) -> Result> { + let content = match tokio::fs::read_to_string(path).await { + Ok(content) => content, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + tracing::info!("No cached resource types at {path}"); + return Ok(None); + } + Err(e) => { + return Err(Error::InternalErr(format!( + "Failed to read cached resource types at {path}: {e}" + ))) + } + }; + let types: Vec = serde_json::from_str(&content).map_err(|e| { + Error::InternalErr(format!( + "Failed to parse cached resource types at {path}: {e}" + )) + })?; + Ok(Some(apply(db, SyncSource::ImageCache, &types).await?)) +} + +/// While `sync_hub_resource_types_daily` is on, syncs once a day, from whichever server first +/// finds the sync due. Off by default: each sync overwrites local edits to the `admins` types +/// the hub also defines. The state it reads lives in the database, so servers starting or +/// restarting do not each sync again. +pub fn spawn_daily_sync(db: DB, mut killpill_rx: tokio::sync::broadcast::Receiver<()>) { + tokio::spawn(async move { + // Leaves time for the hub URL and secret settings to load, and spreads out servers + // that start together. + let mut wait = std::time::Duration::from_secs(rand::random_range(30..90)); + loop { + tokio::select! { + _ = tokio::time::sleep(wait) => {} + _ = killpill_rx.recv() => return, + } + if let Err(e) = sync_if_due(&db).await { + tracing::error!("Hub resource type sync: {e:#}"); + } + wait = CHECK_INTERVAL; + } + }); +} + +async fn sync_if_due(db: &DB) -> Result<()> { + if !setting_enabled(db, SYNC_HUB_RESOURCE_TYPES_DAILY_SETTING).await? + || setting_enabled(db, DISABLE_HUB_SETTING).await? + { + return Ok(()); + } + // A transaction-scoped lock: a dropped transaction rolls back and releases it, so a + // pass cut short cannot leave it held on a pooled connection. The transaction only + // owns the lock; the sync runs on other connections. + let mut lock_tx = db.begin().await?; + let locked: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock($1)") + .bind(SYNC_LOCK_ID) + .fetch_one(&mut *lock_tx) + .await?; + if !locked { + return Ok(()); + } + let state = load_state(db).await?; + let now = Utc::now(); + let due = state + .last_hub_sync_at + .is_none_or(|at| now - at >= SYNC_INTERVAL) + && state + .last_attempt_at + .is_none_or(|at| now - at >= RETRY_INTERVAL); + // Hub only. Falling back to the image's cache here would re-apply the same listing every + // hour on an instance that never reaches a hub, over any edit made since; the cache is + // applied at boot (`SYNC_CACHED_RT`) or on demand instead. + if due { + match sync_from_hub(db).await { + Ok(outcome) if outcome.changed.is_empty() => tracing::info!("{}", outcome.summary()), + Ok(outcome) => { + tracing::info!("{}: {}", outcome.summary(), outcome.changed.join(", ")) + } + Err(e) => tracing::warn!("Hub resource type sync failed, retrying in an hour: {e}"), + } + } + lock_tx.rollback().await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hub_type( + format_extension: Option>, + display_name: Option>, + is_fileset: Option, + ) -> HubResourceType { + HubResourceType { + name: "t".to_string(), + schema: None, + description: None, + format_extension: format_extension.map(|e| e.map(str::to_string)), + display_name: display_name.map(|n| n.map(str::to_string)), + is_fileset, + } + } + + fn stored( + format_extension: Option<&str>, + display_name: Option<&str>, + is_fileset: bool, + ) -> ResourceTypeRow { + ResourceTypeRow { + schema: None, + description: None, + format_extension: format_extension.map(str::to_string), + is_fileset, + display_name: display_name.map(str::to_string), + } + } + + #[test] + fn absent_keys_keep_the_stored_values_and_nulls_clear_them() { + let current = stored(Some("csv"), Some("T"), false); + let kept = target_row(&hub_type(None, None, None), Some(¤t)); + assert_eq!(kept, current); + let cleared = target_row(&hub_type(Some(None), Some(None), None), Some(¤t)); + assert_eq!(cleared, stored(None, None, false)); + } + + #[test] + fn a_fileset_never_takes_a_format_extension() { + let local_fileset = stored(None, None, true); + let row = target_row( + &hub_type(Some(Some("csv")), None, None), + Some(&local_fileset), + ); + assert_eq!(row, stored(None, None, true)); + let row = target_row(&hub_type(Some(Some("csv")), None, Some(true)), None); + assert_eq!(row, stored(None, None, true)); + } + + #[sqlx::test(migrations = "../migrations")] + async fn only_sync_cached_rt_applies_the_image_cache_after_a_hub_sync( + db: DB, + ) -> anyhow::Result<()> { + let dir = tempfile::tempdir()?; + let path = dir.path().join("resource_types.json"); + std::fs::write( + &path, + json!([{ "name": "cached_type", "schema": null, "description": "image copy" }]) + .to_string(), + )?; + let path = path.to_str().unwrap(); + let description = || { + sqlx::query_scalar::<_, Option>( + "SELECT description FROM resource_type WHERE workspace_id = 'admins' AND name = 'cached_type'", + ) + .fetch_one(&db) + }; + + assert!(fall_back_to_image_cache(&db, path).await?.is_some()); + assert_eq!(description().await?.as_deref(), Some("image copy")); + + record_state(&db, json!({ "last_hub_sync_at": Utc::now() })).await?; + sqlx::query( + "UPDATE resource_type SET description = 'hub copy' WHERE workspace_id = 'admins' AND name = 'cached_type'", + ) + .execute(&db) + .await?; + assert!(fall_back_to_image_cache(&db, path).await?.is_none()); + assert_eq!(description().await?.as_deref(), Some("hub copy")); + + assert!(sync_from_cache_file(&db, path).await?.is_some()); + assert_eq!(description().await?.as_deref(), Some("image copy")); + Ok(()) + } + + #[sqlx::test(migrations = "../migrations")] + async fn no_daily_sync_unless_turned_on(db: DB) -> anyhow::Result<()> { + sync_if_due(&db).await?; + + let state: Option = + sqlx::query_scalar("SELECT value FROM background_task_state WHERE name = $1") + .bind(SYNC_STATE_TASK) + .fetch_optional(&db) + .await?; + assert_eq!(state, None); + Ok(()) + } +} diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index 700ba71df5..3d50558c7f 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -267,6 +267,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub disable_hub: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub sync_hub_resource_types_daily: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub auto_build_binary_on_deploy: Option, #[serde(skip_serializing_if = "Option::is_none")] pub ai_sessions_instance_storage_fallback: Option, @@ -281,6 +283,8 @@ pub struct GlobalSettings { #[serde(skip_serializing_if = "Option::is_none")] pub github_app_webhook_base_url: Option, #[serde(skip_serializing_if = "Option::is_none")] + pub api_base_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub email_domain: Option, #[serde(skip_serializing_if = "Option::is_none")] pub hub_base_url: Option, @@ -1375,8 +1379,8 @@ pub fn diff_worker_configs( } /// Declaratively replace the global settings, rejecting a -/// `github_app_webhook_base_url` or `http_route_default_allowed_origins` the -/// API would reject. +/// `github_app_webhook_base_url`, `api_base_url` or +/// `http_route_default_allowed_origins` the API would reject. /// /// Every declarative writer (the `sync-config` CLI, the Kubernetes operator's /// ConfigMap sync) MUST go through this rather than calling @@ -1399,29 +1403,35 @@ pub async fn sync_global_settings_declarative( current: &BTreeMap, desired: &BTreeMap, ) -> anyhow::Result<()> { - let webhook_key = crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING; - // Non-string shapes are rejected rather than ignored: `as_str()` alone would let a - // bool/number/object through as if the key were absent, and the diff below would - // then persist it — where the HTTP path answers "must be a URL string". - match desired.get(webhook_key) { - None | Some(serde_json::Value::Null) => {} - Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} - Some(serde_json::Value::String(s)) => crate::global_settings::validate_webhook_base_url(s) - .map_err(|e| anyhow::anyhow!("{webhook_key}: {e}"))?, - // Names the JSON kind rather than printing it: this is the last message on - // this path that could report submitted content, and an object or array could - // carry a secret into `sync-config` output and operator logs. - Some(other) => { - let kind = match other { - serde_json::Value::Bool(_) => "a boolean", - serde_json::Value::Number(_) => "a number", - serde_json::Value::Array(_) => "an array", - serde_json::Value::Object(_) => "an object", - _ => "a non-string value", - }; - return Err(anyhow::anyhow!( - "{webhook_key} must be a URL string, got {kind}" - )); + for url_key in [ + crate::global_settings::GITHUB_APP_WEBHOOK_BASE_URL_SETTING, + crate::global_settings::API_BASE_URL_SETTING, + ] { + // Non-string shapes are rejected rather than ignored: `as_str()` alone would let a + // bool/number/object through as if the key were absent, and the diff below would + // then persist it — where the HTTP path answers "must be a URL string". + match desired.get(url_key) { + None | Some(serde_json::Value::Null) => {} + Some(serde_json::Value::String(s)) if s.trim().is_empty() => {} + Some(serde_json::Value::String(s)) => { + crate::global_settings::validate_webhook_base_url(s) + .map_err(|e| anyhow::anyhow!("{url_key}: {e}"))? + } + // Names the JSON kind rather than printing it: this is the last message on + // this path that could report submitted content, and an object or array could + // carry a secret into `sync-config` output and operator logs. + Some(other) => { + let kind = match other { + serde_json::Value::Bool(_) => "a boolean", + serde_json::Value::Number(_) => "a number", + serde_json::Value::Array(_) => "an array", + serde_json::Value::Object(_) => "an object", + _ => "a non-string value", + }; + return Err(anyhow::anyhow!( + "{url_key} must be a URL string, got {kind}" + )); + } } } diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index 5754b8c8c1..5ccc8beff1 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -75,6 +75,7 @@ pub mod flow_status; pub mod flows; pub mod folders; pub mod global_settings; +pub mod hub_resource_types; pub mod guest_jwt; pub mod indexer; pub mod instance_config; @@ -123,6 +124,7 @@ pub mod query_builders; pub mod queue; pub mod queue_metrics; pub mod result_stream; +pub mod runnable_job_stats; pub mod runnable_settings; pub mod runnables; pub mod schedule; diff --git a/backend/windmill-common/src/otel_oss.rs b/backend/windmill-common/src/otel_oss.rs index bd0a5202df..95e635dd8f 100644 --- a/backend/windmill-common/src/otel_oss.rs +++ b/backend/windmill-common/src/otel_oss.rs @@ -132,3 +132,36 @@ pub fn otel_set_health_status_phase(_phase: &str) {} #[cfg(not(feature = "private"))] pub fn otel_set_health_db_unresponsive(_unresponsive: bool) {} + +#[cfg(not(feature = "private"))] +pub fn otel_set_worker_memory( + _worker: &str, + _worker_group: &str, + _usage: Option, + _windmill_usage: Option, + _limit: Option, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_add_worker_cpu( + _worker: &str, + _worker_group: &str, + _usage_secs: f64, + _periods: u64, + _throttled_periods: u64, + _throttled_secs: f64, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_set_worker_occupancy_rate( + _worker: &str, + _worker_group: &str, + _window: &'static str, + _rate: f64, +) { +} + +#[cfg(not(feature = "private"))] +pub fn otel_record_job_memory_peak(_tag: &str, _bytes: u64) {} diff --git a/backend/windmill-common/src/runnable_job_stats.rs b/backend/windmill-common/src/runnable_job_stats.rs new file mode 100644 index 0000000000..fa1b99e215 --- /dev/null +++ b/backend/windmill-common/src/runnable_job_stats.rs @@ -0,0 +1,288 @@ +//! Hourly resource rollup per runnable and worker group (`runnable_job_stats`). +//! +//! Workers accumulate completed jobs in memory and upsert periodically, so that the +//! "heaviest scripts" view never has to aggregate `v2_job_completed`. +//! +//! What a row leaves out: executors that run in the worker process (SQL, native +//! TypeScript, ...) have no CPU reading, and for memory report nothing or, for some SQL +//! ones, the size of their result, as the job's `memory_peak` does. Memory is otherwise +//! the peak of the job's main process only, and CPU counts that process and the children +//! it waited for, and nothing for a job that was killed. +//! +//! Nothing here checks authorization: these functions write and prune every workspace's +//! rows, and are for the worker's completion path and the server's monitor only. Reads +//! go through the devops-gated API handler. + +use std::{collections::HashMap, sync::Mutex}; + +use sqlx::{Pool, Postgres}; + +use crate::{jobs::JobKind, worker_group_job_stats::get_current_hour}; + +/// Every run without a stable path is counted under this one, per workspace and worker +/// group: previews and other ad-hoc runs carry caller-chosen paths that would otherwise +/// grow the table without bound. +pub const ADHOC_RUNNABLE_PATH: &str = ""; + +/// Also the furthest back the API reads. +pub const RETENTION_DAYS: i64 = 30; + +#[derive(Debug, Default, Clone, Copy, PartialEq)] +struct RunnableStats { + job_count: i32, + total_duration_ms: i64, + max_memory_peak: i32, + sum_memory_peak: i64, + memory_sample_count: i32, + total_cpu_ms: i64, +} + +impl RunnableStats { + /// `mem_peak` is in kB and non-positive when the job had no reading. + fn add_job(&mut self, duration_ms: i64, mem_peak: i32, cpu_time_ms: Option) { + self.job_count += 1; + self.total_duration_ms += duration_ms; + if mem_peak > 0 { + self.max_memory_peak = self.max_memory_peak.max(mem_peak); + self.sum_memory_peak += mem_peak as i64; + self.memory_sample_count += 1; + } + self.total_cpu_ms += cpu_time_ms.unwrap_or(0).max(0); + } + + fn merge(&mut self, other: &RunnableStats) { + self.job_count += other.job_count; + self.total_duration_ms += other.total_duration_ms; + self.max_memory_peak = self.max_memory_peak.max(other.max_memory_peak); + self.sum_memory_peak += other.sum_memory_peak; + self.memory_sample_count += other.memory_sample_count; + self.total_cpu_ms += other.total_cpu_ms; + } +} + +/// (hour, workspace_id, runnable_path, worker_group) +type StatsKey = (i64, String, String, String); + +lazy_static::lazy_static! { + static ref STATS: Mutex> = Mutex::new(HashMap::new()); +} + +/// The path a job is rolled up under: its own for the kinds whose path names something +/// deployed (a flow step's is `/`), the ad-hoc bucket otherwise. +pub fn stats_path(kind: JobKind, runnable_path: Option<&str>) -> &str { + match (kind, runnable_path) { + ( + JobKind::Script + | JobKind::Script_Hub + | JobKind::FlowScript + | JobKind::FlowNode + | JobKind::AIAgent, + Some(path), + ) if !path.is_empty() => path, + _ => ADHOC_RUNNABLE_PATH, + } +} + +pub fn accumulate_runnable_job_stats( + workspace_id: &str, + kind: JobKind, + runnable_path: Option<&str>, + worker_group: &str, + duration_ms: i64, + mem_peak: i32, + cpu_time_ms: Option, +) { + let key = ( + get_current_hour(), + workspace_id.to_string(), + stats_path(kind, runnable_path).to_string(), + worker_group.to_string(), + ); + STATS + .lock() + .unwrap() + .entry(key) + .or_default() + .add_job(duration_ms, mem_peak, cpu_time_ms); +} + +/// Counts the time spent on a round of a job that is not completing, without counting a +/// job: a suspended workflow-as-code round replays its finished steps for free when it +/// resumes, so the time they took would otherwise never reach the rollup. A row can +/// therefore hold time and no job until the workflow completes. +pub fn accumulate_runnable_round( + workspace_id: &str, + kind: JobKind, + runnable_path: Option<&str>, + worker_group: &str, + duration_ms: i64, + cpu_time_ms: Option, +) { + let key = ( + get_current_hour(), + workspace_id.to_string(), + stats_path(kind, runnable_path).to_string(), + worker_group.to_string(), + ); + let mut stats = STATS.lock().unwrap(); + let entry = stats.entry(key).or_default(); + entry.total_duration_ms += duration_ms; + entry.total_cpu_ms += cpu_time_ms.unwrap_or(0).max(0); +} + +pub async fn flush_runnable_job_stats(db: &Pool) -> Result<(), sqlx::Error> { + let drained: Vec<(StatsKey, RunnableStats)> = { + let mut stats = STATS.lock().unwrap(); + if stats.is_empty() { + return Ok(()); + } + stats.drain().collect() + }; + + let mut hours = Vec::with_capacity(drained.len()); + let mut workspace_ids = Vec::with_capacity(drained.len()); + let mut runnable_paths = Vec::with_capacity(drained.len()); + let mut worker_groups = Vec::with_capacity(drained.len()); + let mut job_counts = Vec::with_capacity(drained.len()); + let mut durations = Vec::with_capacity(drained.len()); + let mut max_peaks = Vec::with_capacity(drained.len()); + let mut sum_peaks = Vec::with_capacity(drained.len()); + let mut sample_counts = Vec::with_capacity(drained.len()); + let mut cpus = Vec::with_capacity(drained.len()); + for ((hour, workspace_id, runnable_path, worker_group), s) in &drained { + hours.push(*hour); + workspace_ids.push(workspace_id.clone()); + runnable_paths.push(runnable_path.clone()); + worker_groups.push(worker_group.clone()); + job_counts.push(s.job_count); + durations.push(s.total_duration_ms); + max_peaks.push(s.max_memory_peak); + sum_peaks.push(s.sum_memory_peak); + sample_counts.push(s.memory_sample_count); + cpus.push(s.total_cpu_ms); + } + + // The join drops rows of a workspace deleted since its jobs ran. The ORDER BY makes + // every worker lock the rows they share in one order: without it two concurrent + // flushes can deadlock, and the join is free to undo an ordering of the arrays. + let res = sqlx::query!( + r#" + INSERT INTO runnable_job_stats + (hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms, + max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms) + SELECT s.hour, s.workspace_id, s.runnable_path, s.worker_group, s.job_count, + s.total_duration_ms, s.max_memory_peak, s.sum_memory_peak, + s.memory_sample_count, s.total_cpu_ms + FROM UNNEST($1::bigint[], $2::text[], $3::text[], $4::text[], $5::int[], $6::bigint[], + $7::int[], $8::bigint[], $9::int[], $10::bigint[]) + AS s(hour, workspace_id, runnable_path, worker_group, job_count, total_duration_ms, + max_memory_peak, sum_memory_peak, memory_sample_count, total_cpu_ms) + JOIN workspace w ON w.id = s.workspace_id + ORDER BY s.hour, s.workspace_id, s.runnable_path, s.worker_group + ON CONFLICT (hour, workspace_id, runnable_path, worker_group) DO UPDATE SET + job_count = runnable_job_stats.job_count + EXCLUDED.job_count, + total_duration_ms = runnable_job_stats.total_duration_ms + EXCLUDED.total_duration_ms, + max_memory_peak = GREATEST(runnable_job_stats.max_memory_peak, EXCLUDED.max_memory_peak), + sum_memory_peak = runnable_job_stats.sum_memory_peak + EXCLUDED.sum_memory_peak, + memory_sample_count = runnable_job_stats.memory_sample_count + EXCLUDED.memory_sample_count, + total_cpu_ms = runnable_job_stats.total_cpu_ms + EXCLUDED.total_cpu_ms + "#, + &hours, + &workspace_ids, + &runnable_paths, + &worker_groups, + &job_counts, + &durations, + &max_peaks, + &sum_peaks, + &sample_counts, + &cpus, + ) + .execute(db) + .await; + + if let Err(e) = res { + let mut stats = STATS.lock().unwrap(); + for (key, s) in drained { + stats.entry(key).or_default().merge(&s); + } + return Err(e); + } + Ok(()) +} + +/// Also removes the rows of workspaces that no longer exist: a flush that overlaps a +/// workspace's deletion still sees the workspace and can write after the deletion's own +/// cleanup ran. Such a row is at most a couple of hours old, so only the last day is +/// searched, which keeps both statements on a range of the primary key. +pub async fn cleanup_old_runnable_job_stats( + db: &Pool, + retention_days: i64, +) -> Result { + let current_hour = get_current_hour(); + let expired = sqlx::query!( + "DELETE FROM runnable_job_stats WHERE hour < $1", + current_hour - retention_days * 24 * 3600 + ) + .execute(db) + .await?; + let orphaned = sqlx::query!( + "DELETE FROM runnable_job_stats s WHERE s.hour >= $1 \ + AND NOT EXISTS (SELECT 1 FROM workspace w WHERE w.id = s.workspace_id)", + current_hour - 24 * 3600 + ) + .execute(db) + .await?; + Ok(expired.rows_affected() + orphaned.rows_affected()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_deployed_runnables_keep_their_path() { + assert_eq!( + stats_path(JobKind::Script, Some("f/etl/load")), + "f/etl/load" + ); + assert_eq!( + stats_path(JobKind::FlowScript, Some("f/etl/flow/a")), + "f/etl/flow/a" + ); + // A preview carries the path being edited, which is not a stable identity. + assert_eq!( + stats_path(JobKind::Preview, Some("f/etl/load")), + ADHOC_RUNNABLE_PATH + ); + assert_eq!( + stats_path(JobKind::FlowPreview, Some("f/etl/flow")), + ADHOC_RUNNABLE_PATH + ); + assert_eq!( + stats_path(JobKind::Dependencies, Some("f/etl/load")), + ADHOC_RUNNABLE_PATH + ); + assert_eq!(stats_path(JobKind::Script, None), ADHOC_RUNNABLE_PATH); + } + + #[test] + fn jobs_without_a_memory_reading_do_not_dilute_it() { + let mut s = RunnableStats::default(); + s.add_job(1000, 2048, Some(700)); + s.add_job(500, -3, None); + s.add_job(250, 0, Some(-5)); + s.add_job(250, 1024, Some(50)); + assert_eq!( + s, + RunnableStats { + job_count: 4, + total_duration_ms: 2000, + max_memory_peak: 2048, + sum_memory_peak: 3072, + memory_sample_count: 2, + total_cpu_ms: 750, + } + ); + } +} diff --git a/backend/windmill-common/src/variables.rs b/backend/windmill-common/src/variables.rs index 1feb4182ab..f45d457093 100644 --- a/backend/windmill-common/src/variables.rs +++ b/backend/windmill-common/src/variables.rs @@ -14,7 +14,7 @@ use crate::utils::WarnAfterExt; use crate::worker::Connection; use crate::{worker::WORKER_GROUP, BASE_URL, DB}; use chrono::{SecondsFormat, Utc}; -use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait}; +use magic_crypt::{MagicCrypt256, MagicCryptTrait}; use quick_cache::sync::Cache; use serde::{Deserialize, Serialize}; @@ -375,13 +375,15 @@ pub fn encrypt(mc: &MagicCrypt256, value: &str) -> String { mc.encrypt_str_to_base64(value) } +const DECRYPT_FAILED: &str = + "Could not decrypt value. The value may have been encrypted with a different key."; + +/// Every failure returns the same message so responses carry no signal about why decryption +/// failed (padding, base64, UTF-8); the cause is only logged. pub fn decrypt(mc: &MagicCrypt256, value: String) -> error::Result { - mc.decrypt_base64_to_string(value).map_err(|e| match e { - MagicCryptError::DecryptError(_) => error::Error::internal_err( - "Could not decrypt value. The value may have been encrypted with a different key." - .to_string(), - ), - _ => error::Error::internal_err(e.to_string()), + mc.decrypt_base64_to_string(value).map_err(|e| { + tracing::warn!("decryption failed: {e}"); + error::Error::internal_err(DECRYPT_FAILED.to_string()) }) } @@ -595,7 +597,7 @@ pub async fn get_reserved_variables( ContextualVariable { name: "WM_END_USER_EMAIL".to_string(), value: end_user_email.unwrap_or_else(|| "".to_string()), - description: "Email of the end user that executed the current script, propagated to flow steps. Only set when the run was triggered from an app (empty otherwise).".to_string(), + description: "Email of the end user that executed the current script, even when it runs on behalf of another identity, propagated to flow steps and child jobs. Set for app and API runs (for a webhook, the owner of the token it was called with), empty for runs fired by a schedule or by a trigger such as an HTTP route or Kafka.".to_string(), is_custom: false, }, ContextualVariable { @@ -746,7 +748,31 @@ pub async fn get_variable_or_self_as( #[cfg(test)] mod tests { - use super::{can_bind_as_prologue_const, escape_js_single_quoted, is_valid_js_identifier}; + use super::{ + can_bind_as_prologue_const, decrypt, escape_js_single_quoted, is_valid_js_identifier, + DECRYPT_FAILED, + }; + use crate::error::Error; + use magic_crypt::MagicCryptTrait; + + #[test] + fn decrypt_failures_all_return_the_same_message() { + let mc = magic_crypt::new_magic_crypt!("workspace-key", 256); + let other = magic_crypt::new_magic_crypt!("another-key", 256); + let failures = [ + "not base64 at all!".to_string(), + mc.encrypt_bytes_to_base64(&[0xff, 0xfe, 0xfd]), + other.encrypt_str_to_base64("secret"), + ]; + for value in failures { + match decrypt(&mc, value.clone()) { + Err(Error::InternalErrLoc { message, .. }) => { + assert_eq!(message, DECRYPT_FAILED, "for {value:?}") + } + other => panic!("expected the uniform decrypt error for {value:?}, got {other:?}"), + } + } + } #[test] fn is_valid_js_identifier_gates_prologue_injection() { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 52fa2da905..58223c5d4a 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -1201,6 +1201,8 @@ pub struct PythonAnnotations { pub py311: bool, pub py312: bool, pub py313: bool, + pub py314: bool, + pub py315: bool, pub sandbox: bool, } @@ -1783,6 +1785,76 @@ pub fn get_cpu_period() -> Option { Some(100000) } +/// Bucket bounds in bytes, 16 MiB to 32 GiB, shared by the Prometheus and OTel job +/// peak-memory histograms so the two exports stay comparable. +pub const JOB_MEMORY_PEAK_BUCKETS: [f64; 12] = { + let mut buckets = [0.0; 12]; + let mut i = 0; + while i < 12 { + buckets[i] = (16u64 << (20 + i)) as f64; + i += 1; + } + buckets +}; + +/// Cumulative CPU accounting of the worker's cgroup. The throttling fields stay at 0 +/// when the cgroup has no CPU controller, which is also what an unthrottled one reports. +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct CpuStat { + pub usage_usec: u64, + pub nr_periods: u64, + pub nr_throttled: u64, + pub throttled_usec: u64, +} + +#[cfg(not(windows))] +fn cpu_stat_field(stat: &str, key: &str) -> Option { + stat.lines().find_map(|line| { + let (k, v) = line.split_once(' ')?; + (k == key).then(|| v.trim().parse().ok()).flatten() + }) +} + +#[cfg(not(windows))] +fn parse_cgroup_v2_cpu_stat(stat: &str) -> Option { + Some(CpuStat { + usage_usec: cpu_stat_field(stat, "usage_usec")?, + nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0), + nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0), + throttled_usec: cpu_stat_field(stat, "throttled_usec").unwrap_or(0), + }) +} + +/// cgroup v1 splits the same figures over two controllers and counts time in nanoseconds. +#[cfg(not(windows))] +fn parse_cgroup_v1_cpu_stat(cpuacct_usage: &str, stat: &str) -> Option { + Some(CpuStat { + usage_usec: cpuacct_usage.trim().parse::().ok()? / 1000, + nr_periods: cpu_stat_field(stat, "nr_periods").unwrap_or(0), + nr_throttled: cpu_stat_field(stat, "nr_throttled").unwrap_or(0), + throttled_usec: cpu_stat_field(stat, "throttled_time").unwrap_or(0) / 1000, + }) +} + +#[cfg(not(windows))] +pub fn get_cpu_stat() -> Option { + const V1_USAGE: &str = "/sys/fs/cgroup/cpuacct/cpuacct.usage"; + if Path::new(V1_USAGE).exists() { + parse_cgroup_v1_cpu_stat( + &std::fs::read_to_string(V1_USAGE).ok()?, + &std::fs::read_to_string("/sys/fs/cgroup/cpu/cpu.stat").unwrap_or_default(), + ) + } else { + let cgroup_path = get_cgroupv2_path()?; + parse_cgroup_v2_cpu_stat(&std::fs::read_to_string(format!("{cgroup_path}/cpu.stat")).ok()?) + } +} + +#[cfg(windows)] +pub fn get_cpu_stat() -> Option { + None +} + /// CPUs the process is allowed to run on, ignoring any bandwidth quota — the count /// Go's `NumCPU` reports. `available_parallelism` cannot stand in for it: that folds /// the quota in, so a fraction of a CPU makes it report a single-core machine. @@ -2822,6 +2894,8 @@ pub enum PyVAlias { #[default] Py312, Py313, + Py314, + Py315, } impl Into for PyVAlias { @@ -2839,13 +2913,20 @@ impl Into for PyVAlias { impl PyVAlias { pub fn all>() -> Vec { use PyVAlias::*; - vec![Py310.into(), Py311.into(), Py312.into(), Py313.into()] + vec![ + Py310.into(), + Py311.into(), + Py312.into(), + Py313.into(), + Py314.into(), + Py315.into(), + ] } // Get MAJOR part of alias. (semver: MAJOR.MINOR.PATCH) fn major(&self) -> u32 { use PyVAlias::*; match self { - Py310 | Py311 | Py312 | Py313 => 3, + Py310 | Py311 | Py312 | Py313 | Py314 | Py315 => 3, // Py400 | Py401 => 4 } } @@ -2860,6 +2941,8 @@ impl PyVAlias { "311" => Some(Py311), "312" => Some(Py312), "313" => Some(Py313), + "314" => Some(Py314), + "315" => Some(Py315), _ => None, } } @@ -2919,6 +3002,33 @@ mod tests { use super::*; use std::collections::HashMap; + #[cfg(not(windows))] + #[test] + fn cgroup_cpu_stat_is_normalized_to_microseconds() { + let v2 = "usage_usec 2500000\nuser_usec 2000000\nsystem_usec 500000\n\ + nr_periods 40\nnr_throttled 7\nthrottled_usec 350000\nnr_bursts 0\n"; + let expected = CpuStat { + usage_usec: 2_500_000, + nr_periods: 40, + nr_throttled: 7, + throttled_usec: 350_000, + }; + assert_eq!(parse_cgroup_v2_cpu_stat(v2), Some(expected)); + + let v1_stat = "nr_periods 40\nnr_throttled 7\nthrottled_time 350000000\n"; + assert_eq!( + parse_cgroup_v1_cpu_stat("2500000000\n", v1_stat), + Some(expected) + ); + + // No CPU controller: usage is still reported, throttling reads as zero. + assert_eq!( + parse_cgroup_v2_cpu_stat("usage_usec 12\nuser_usec 8\nsystem_usec 4\n"), + Some(CpuStat { usage_usec: 12, ..Default::default() }) + ); + assert_eq!(parse_cgroup_v2_cpu_stat("nr_periods 40\n"), None); + } + #[test] fn datatable_role_is_read_from_the_leading_comment_block() { let role = |code| SqlAnnotations::datatable_role(code); diff --git a/backend/windmill-common/src/workspaces.rs b/backend/windmill-common/src/workspaces.rs index 7baad65a60..f819c5b796 100644 --- a/backend/windmill-common/src/workspaces.rs +++ b/backend/windmill-common/src/workspaces.rs @@ -1067,7 +1067,7 @@ pub async fn billable_seats(db: &crate::DB, w_id: &str) -> Result .fetch_one(db) .await .map_err(|e| Error::internal_err(format!("counting billable seats of {w_id}: {e:#}")))?; - let (developers, operators) = if operator_can_build_flows(db, w_id).await? { + let (developers, operators) = if operator_builder_rights(db, w_id).await?.any() { (row.developers + row.operators, 0) } else { (row.developers, row.operators) @@ -1205,14 +1205,47 @@ lazy_static::lazy_static! { /// Every operator right of a workspace, read and cached together because they share one /// `operator_settings` column and one invalidation. The two groups have opposite polarity: -/// `builder_flows` is granted on request and costs a seat, the `manage` rights are held by +/// the builder rights are granted on request and cost a seat, the `manage` rights are held by /// default and cost nothing. #[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] pub struct OperatorRights { - pub builder_flows: bool, + pub builder: OperatorBuilderRights, pub manage: OperatorManageRights, } +/// The two things a workspace can let its operators compose. Granted independently: composing a +/// flow is fully checked server-side, while a full-code app is a browser-built bundle no check can +/// read, so an admin may reasonably want the first without the second. +#[derive(Copy, Clone, Debug, Default, PartialEq, Eq)] +pub struct OperatorBuilderRights { + pub flows: bool, + pub apps: bool, +} + +/// Which capability a gate needs. Naming a kind at each call site is what keeps the two rights +/// from collapsing back into one: a gate that takes "either" authorizes an app write on a +/// flows-only workspace. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum BuilderKind { + Flows, + Apps, +} + +impl OperatorBuilderRights { + /// Whether either right is granted. Only for surfaces that are genuinely not per-kind: the + /// seat a membership costs, and listing drafts. + pub fn any(&self) -> bool { + self.flows || self.apps + } + + pub fn has(&self, kind: BuilderKind) -> bool { + match kind { + BuilderKind::Flows => self.flows, + BuilderKind::Apps => self.apps, + } + } +} + /// Writes an operator may perform unless the workspace withdraws them. Unlike the visibility /// flags beside them in `operator_settings`, these are enforced; unlike a right a workspace /// grants, they are held by default and cost no seat. @@ -1291,6 +1324,7 @@ async fn operator_rights(db: &DB, workspace_id: &str) -> Result // defaults have to differ here rather than at the call sites. let row = sqlx::query!( "SELECT COALESCE((operator_settings->>'builder_flows')::boolean, false) AS \"flows!\", + COALESCE((operator_settings->>'builder_apps')::boolean, false) AS \"apps!\", COALESCE((operator_settings->>'manage_schedules')::boolean, true) AS \"schedules!\", COALESCE((operator_settings->>'manage_triggers')::boolean, true) AS \"triggers!\" FROM workspace_settings WHERE workspace_id = $1", @@ -1306,7 +1340,7 @@ async fn operator_rights(db: &DB, workspace_id: &str) -> Result let rights = row .map(|r| OperatorRights { - builder_flows: r.flows, + builder: OperatorBuilderRights { flows: r.flows, apps: r.apps }, manage: OperatorManageRights { schedules: r.schedules, triggers: r.triggers }, }) .unwrap_or_default(); @@ -1316,21 +1350,22 @@ async fn operator_rights(db: &DB, workspace_id: &str) -> Result Ok(rights) } -/// Whether operators of this workspace may compose flows out of already-deployed runnables. Per -/// workspace, not per user: every operator gets it, and consumes a full seat for it. -pub async fn operator_can_build_flows(db: &DB, workspace_id: &str) -> Result { - Ok(operator_rights(db, workspace_id).await?.builder_flows) +/// What operators of this workspace may compose out of already-deployed runnables. Per workspace, +/// not per user: every operator gets whatever is granted, and consumes a full seat for it. +pub async fn operator_builder_rights(db: &DB, workspace_id: &str) -> Result { + Ok(operator_rights(db, workspace_id).await?.builder) } -/// Gate for a write only a workspace that granted builder rights lets operators perform. `action` -/// completes "Operators cannot {action} for security reasons". -pub async fn check_operator_can_build_flows( +/// Gate for a write only a workspace that granted the builder right covering `kind` lets operators +/// perform. `action` completes "Operators cannot {action} for security reasons". +pub async fn check_operator_can_build( db: &DB, workspace_id: &str, is_operator: bool, + kind: BuilderKind, action: &str, ) -> Result<()> { - if is_operator && !operator_can_build_flows(db, workspace_id).await? { + if is_operator && !operator_builder_rights(db, workspace_id).await?.has(kind) { return Err(Error::PermissionDenied(format!( "Operators cannot {action} for security reasons" ))); @@ -1338,14 +1373,15 @@ pub async fn check_operator_can_build_flows( Ok(()) } -/// Whether a membership consumes an operator (half) seat rather than an author seat. A builder -/// right makes an operator an author of deployable artifacts, so it weighs a full seat. +/// Whether a membership consumes an operator (half) seat rather than an author seat. Either +/// builder right makes an operator an author of deployable artifacts, so both weigh the same full +/// seat: there is no half-author. pub async fn consumes_operator_seat( db: &DB, workspace_id: &str, is_operator: bool, ) -> Result { - Ok(is_operator && !operator_can_build_flows(db, workspace_id).await?) + Ok(is_operator && !operator_builder_rights(db, workspace_id).await?.any()) } /// Invalidate the operator rights cache for a workspace diff --git a/backend/windmill-common/tests/billing_workspace.rs b/backend/windmill-common/tests/billing_workspace.rs index 0eddeabf53..ccc013d47a 100644 --- a/backend/windmill-common/tests/billing_workspace.rs +++ b/backend/windmill-common/tests/billing_workspace.rs @@ -143,6 +143,20 @@ async fn paid_seats_and_fork_count(db: Pool) { (4, 0, 1, 5), "builder rights bill every operator as a developer, service accounts stay half a seat" ); + sqlx::query( + "UPDATE workspace_settings SET operator_settings = '{\"builder_apps\": true}' + WHERE workspace_id = 'seat-root'", + ) + .execute(&db) + .await + .unwrap(); + windmill_common::workspaces::invalidate_operator_rights_cache("seat-root"); + let breakdown = billable_seats(&db, "seat-root").await.unwrap(); + assert_eq!( + (breakdown.developers, breakdown.operators, breakdown.seats), + (4, 0, 5), + "the app right alone also bills every operator as a developer" + ); insert_ws(&db, "seat-fork1", Some("seat-root"), false).await; insert_ws(&db, "seat-fork2", Some("seat-root"), false).await; diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index e8e3716d2b..68749d2b90 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -182,6 +182,15 @@ pub struct CanceledBy { pub reason: Option, } +/// What only the worker that ran a job knows about it, for the per-runnable rollup: an +/// agent worker's completion is processed by the server, whose own worker group is not +/// the one the job ran in. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct JobResourceUsage { + pub worker_group: String, + pub cpu_time_ms: Option, +} + #[derive(Debug, Serialize, Deserialize)] pub struct JobCompleted { pub job: MiniCompletedJob, @@ -196,6 +205,8 @@ pub struct JobCompleted { pub duration: Option, pub has_stream: Option, pub from_cache: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resource_usage: Option, #[serde(skip)] pub flow_runners: Option>, #[serde(skip)] @@ -1221,7 +1232,7 @@ pub async fn add_completed_job_pre_shaped_failure, -) -> Result<(), Error> { +) -> Result { record_failure_metrics(completed_job, worker_name).await; tracing::error!( @@ -1229,7 +1240,7 @@ pub async fn add_completed_job_pre_shaped_failure, ) -> Result { + add_completed_job_error_with_span( + db, + completed_job, + mem_peak, + canceled_by, + e, + worker_name, + flow_is_done, + duration, + ) + .await + .map(|(wrapped, _)| wrapped) +} + +/// `add_completed_job_error`, also handing back what the completed row holds. +pub async fn add_completed_job_error_with_span( + db: &Pool, + completed_job: &MiniCompletedJob, + mem_peak: i32, + canceled_by: Option, + e: serde_json::Value, + worker_name: &str, + flow_is_done: bool, + duration: Option, +) -> Result<(WrappedError, CompletedSpan), Error> { record_failure_metrics(completed_job, worker_name).await; let result = WrappedError { error: e }; @@ -1266,7 +1302,7 @@ pub async fn add_completed_job_error( completed_job.workspace_id, serde_json::to_string(&result).unwrap_or_else(|_| "".to_string()) ); - let _ = add_completed_job( + let (_, span) = add_completed_job( db, &completed_job, false, @@ -1281,7 +1317,7 @@ pub async fn add_completed_job_error( ) .warn_after_seconds(10) .await?; - Ok(result) + Ok((result, span)) } lazy_static::lazy_static! { @@ -1320,7 +1356,7 @@ pub async fn add_completed_job( flow_is_done: bool, duration: Option, from_cache: bool, -) -> Result<(Uuid, i64), Error> { +) -> Result<(Uuid, CompletedSpan), Error> { // tracing::error!("Start"); // let start = tokio::time::Instant::now(); @@ -1354,7 +1390,7 @@ pub async fn add_completed_job( }; let result_columns = result_columns.as_ref(); - let (opt_uuid, duration, _skip_downstream_error_handlers, wac_parent_ready) = (|| { + let (opt_uuid, span, _skip_downstream_error_handlers, wac_parent_ready) = (|| { commit_completed_job( db, completed_job, @@ -1395,7 +1431,7 @@ pub async fn add_completed_job( // if scheduling next job failed, return the job_id early to ensure the job get retried after a timeout if let Some(job_id) = opt_uuid { - return Ok((job_id, duration)); + return Ok((job_id, span)); } // Auto-resolve a retry chain that ultimately worked, from whichever of the two @@ -1436,7 +1472,7 @@ pub async fn add_completed_job( } #[cfg(feature = "cloud")] - apply_completed_job_cloud_usage(db, completed_job, duration); + apply_completed_job_cloud_usage(db, completed_job, span.last_run_ms); #[cfg(all(feature = "enterprise", feature = "private"))] crate::jobs_ee::apply_completed_job_error_handlers( @@ -1453,7 +1489,7 @@ pub async fn add_completed_job( // tracing::error!("4 {:?}", start.elapsed()); - Ok((completed_job.id, duration)) + Ok((completed_job.id, span)) } async fn commit_completed_job( @@ -1471,7 +1507,7 @@ async fn commit_completed_job( // True when a native script retry was enqueued for this failed attempt, i.e. // this is not the terminal attempt — schedule completion handlers must wait. retry_pending: bool, -) -> windmill_common::error::Result<(Option, i64, bool, bool)> { +) -> windmill_common::error::Result<(Option, CompletedSpan, bool, bool)> { // let start = std::time::Instant::now(); let job_id = completed_job.id; @@ -1538,11 +1574,11 @@ async fn commit_completed_job( && monitor_parent.is_none() && !is_scheduled { - let Some(duration) = completion.execute(&mut *db.acquire().await?).await? else { + let Some(completed) = completion.execute(&mut *db.acquire().await?).await? else { return Err(not_in_queue_error(db, job_id).await); }; - log_completed_job(completed_job, duration, success); - return Ok((None, duration, false, false)); + log_completed_job(completed_job, completed.duration_ms, success); + return Ok((None, completed, false, false)); } let mut tx = db.begin().warn_after_seconds(10).await?; @@ -1550,7 +1586,8 @@ async fn commit_completed_job( // The parent's rows are locked ahead of the child's own queue row (see // `record_child_completion` for the order this must keep), so the duration it stamps is read // before the completion: the one the completed row will hold. `now()` is fixed for the - // transaction, and a completed row already there keeps its own duration. + // transaction, and a completed row already there keeps its own duration. For a child that + // itself parked it is the last run's, matching the last start its timeline entry carries. let mut wac_parent_ready = false; if let Some(parent_job) = wac_parent { let Some(duration) = sqlx::query_scalar!( @@ -1577,7 +1614,7 @@ async fn commit_completed_job( .await?; } - let Some(duration) = completion.execute(&mut *tx).await? else { + let Some(completed) = completion.execute(&mut *tx).await? else { return Err(not_in_queue_error(&mut *tx, job_id).await); }; @@ -1787,16 +1824,35 @@ async fn commit_completed_job( tx.commit().warn_after_seconds(10).await?; - log_completed_job(completed_job, duration, success); + log_completed_job(completed_job, completed.duration_ms, success); // tracing::info!("completed job: {:?}", start.elapsed().as_micros()); Ok(( None, - duration, + completed, _skip_downstream_error_handlers, wac_parent_ready, )) } +/// What a job's completion wrote. `started_at` and `duration_ms` are the completed row's. +/// `last_run_ms` is the time the job's last run held a worker, which is what gets metered; +/// it differs from `duration_ms` only for a job that parked. +#[derive(Debug, Clone, Copy)] +pub struct CompletedSpan { + pub started_at: Option>, + pub duration_ms: i64, + pub last_run_ms: i64, +} + +impl CompletedSpan { + /// Whether the row spans more than the run that started at `last_started_at`, i.e. the + /// job parked and resumed. A flow records such a step with the row's span, so the step + /// reads like the job does. + pub fn outlasts_run(&self, last_started_at: Option>) -> bool { + matches!((self.started_at, last_started_at), (Some(first), Some(last)) if first < last) + } +} + /// What a job's completion writes, for `Completion::execute`. struct Completion<'a> { completed_job: &'a MiniCompletedJob, @@ -1813,6 +1869,11 @@ impl Completion<'_> { /// Moves the job from the queue to the completed jobs and refreshes a flow step's parent /// ping, as one statement. Returns `None` when the job was no longer in the queue. /// + /// For a job that parked (a Workflow-as-Code parent, see `suspend_wac_parent`) the row's + /// `started_at` and `duration_ms` span the whole workflow from its first start, and the + /// last run goes to `extras.wac_last_segment_ms` for readers that sum worker time from + /// completed rows. + /// /// The completion takes its cancellation from the queue row it deletes, not only from /// `canceled_by`: that is what the worker last read, and the delete waits for a cancel still /// being written, so the deleted row is the final word on whether the job was canceled. @@ -1820,7 +1881,7 @@ impl Completion<'_> { /// It locks the queue row before the completed row's key. Any other writer completing a job /// (the monitor's zombie fallback, debounce) must take them in the same order, or the two /// deadlock. - async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result> { + async fn execute(&self, conn: &mut sqlx::PgConnection) -> error::Result> { let Completion { completed_job, success, @@ -1851,16 +1912,20 @@ impl Completion<'_> { .then_some(completed_job.parent_job) .flatten() else { - return sqlx::query_scalar!( + return sqlx::query_as!( + CompletedSpan, "WITH deleted AS ( DELETE FROM v2_job_queue WHERE id = $1 - RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason + RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason, + (extras->>'wac_first_started_at')::timestamptz AS first_started_at, + COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms ), completed AS ( INSERT INTO v2_job_completed AS cj ( workspace_id , id , started_at , duration_ms + , extras , result , result_columns , canceled_by @@ -1871,8 +1936,11 @@ impl Completion<'_> { , status , worker ) - SELECT d.workspace_id, d.id, d.started_at, - COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000), + SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at), + CASE WHEN d.first_started_at IS NULL THEN d.run_ms + ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END, + CASE WHEN d.first_started_at IS NOT NULL + THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END, $3::text::jsonb, $10, CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END, CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END, @@ -1888,9 +1956,11 @@ impl Completion<'_> { THEN EXCLUDED.canceled_by ELSE cj.canceled_by END, canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END - RETURNING duration_ms + RETURNING cj.started_at, cj.duration_ms, + COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms ) - SELECT duration_ms AS \"duration_ms!\" FROM completed", + SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\", + last_run_ms AS \"last_run_ms!\" FROM completed", /* $1 */ completed_job.id, /* $2 */ success, /* $3 */ result, @@ -1910,16 +1980,20 @@ impl Completion<'_> { // A canceled flow is pinged too: it is completed by its next transition like any other // flow, and the zombie flow monitor needs the ping to finish the cancel if that // transition is lost. - sqlx::query_scalar!( + sqlx::query_as!( + CompletedSpan, "WITH deleted AS ( DELETE FROM v2_job_queue WHERE id = $1 - RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason + RETURNING id, workspace_id, started_at, worker, canceled_by, canceled_reason, + (extras->>'wac_first_started_at')::timestamptz AS first_started_at, + COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(started_at, now()))))*1000)::bigint AS run_ms ), completed AS ( INSERT INTO v2_job_completed AS cj ( workspace_id , id , started_at , duration_ms + , extras , result , result_columns , canceled_by @@ -1930,8 +2004,11 @@ impl Completion<'_> { , status , worker ) - SELECT d.workspace_id, d.id, d.started_at, - COALESCE($9::bigint, (EXTRACT('epoch' FROM (now())) - EXTRACT('epoch' FROM (COALESCE(d.started_at, now()))))*1000), + SELECT d.workspace_id, d.id, COALESCE(d.first_started_at, d.started_at), + CASE WHEN d.first_started_at IS NULL THEN d.run_ms + ELSE EXTRACT('epoch' FROM (now() - d.first_started_at))*1000 END, + CASE WHEN d.first_started_at IS NOT NULL + THEN jsonb_build_object('wac_last_segment_ms', d.run_ms) END, $3::text::jsonb, $10, CASE WHEN $4::BOOL THEN $5 ELSE d.canceled_by END, CASE WHEN $4::BOOL THEN $6 WHEN d.canceled_by IS NOT NULL THEN d.canceled_reason END, @@ -1947,14 +2024,16 @@ impl Completion<'_> { THEN EXCLUDED.canceled_by ELSE cj.canceled_by END, canceled_reason = CASE WHEN NOT $4::BOOL AND EXCLUDED.canceled_by IS NOT NULL THEN EXCLUDED.canceled_reason ELSE cj.canceled_reason END - RETURNING duration_ms + RETURNING cj.started_at, cj.duration_ms, + COALESCE((cj.extras->>'wac_last_segment_ms')::bigint, cj.duration_ms) AS last_run_ms ), parent_ping AS ( UPDATE v2_job_runtime r SET ping = now() FROM v2_job_queue q WHERE r.id = $11 AND q.id = r.id AND q.workspace_id = $12 AND EXISTS (SELECT 1 FROM completed) ) - SELECT duration_ms AS \"duration_ms!\" FROM completed", + SELECT started_at AS \"started_at?\", duration_ms AS \"duration_ms!\", + last_run_ms AS \"last_run_ms!\" FROM completed", /* $1 */ completed_job.id, /* $2 */ success, /* $3 */ result, @@ -2021,7 +2100,7 @@ async fn check_result_size( db: &Pool, queued_job: &MiniCompletedJob, result: Json<&T>, -) -> Option, i64, bool, bool), Error>> { +) -> Option, CompletedSpan, bool, bool), Error>> { let result_size = result.size() / 1024 / 1024; if result_size > 2 { if result_size > *MAX_RESULT_SIZE_MB { @@ -4252,6 +4331,7 @@ impl PulledJobResult { ))), result_columns: None, mem_peak: 0, + resource_usage: None, cached_res_path: None, token: "".to_string(), canceled_by: None, @@ -4273,6 +4353,7 @@ impl PulledJobResult { }))), result_columns: None, mem_peak: 0, + resource_usage: None, cached_res_path: None, token: "".to_string(), canceled_by: None, diff --git a/backend/windmill-runtime-nativets/Cargo.toml b/backend/windmill-runtime-nativets/Cargo.toml index d0b4e97f19..22da97ab1d 100644 --- a/backend/windmill-runtime-nativets/Cargo.toml +++ b/backend/windmill-runtime-nativets/Cargo.toml @@ -22,16 +22,15 @@ deno_webidl.workspace = true deno_web.workspace = true deno_crypto.workspace = true deno_net.workspace = true -deno_console.workspace = true -deno_url.workspace = true deno_core.workspace = true deno_ast.workspace = true deno_tls.workspace = true deno_permissions.workspace = true -deno_io.workspace = true -deno_fs.workspace = true deno_telemetry.workspace = true deno_error.workspace = true +sys_traits.workspace = true +# Not used directly: depending on it is what makes the workspace pin bind. +v8.workspace = true winapi.workspace = true itertools.workspace = true @@ -58,14 +57,8 @@ deno_webidl.workspace = true deno_web.workspace = true deno_crypto.workspace = true deno_net.workspace = true -deno_console.workspace = true -deno_url.workspace = true deno_core.workspace = true deno_ast.workspace = true -deno_tls.workspace = true -deno_permissions.workspace = true -deno_io.workspace = true -deno_fs.workspace = true deno_telemetry.workspace = true deno_error.workspace = true winapi.workspace = true diff --git a/backend/windmill-runtime-nativets/build.rs b/backend/windmill-runtime-nativets/build.rs index 29a6dbf81b..fab6cbb1ed 100644 --- a/backend/windmill-runtime-nativets/build.rs +++ b/backend/windmill-runtime-nativets/build.rs @@ -1,107 +1,11 @@ use deno_ast::{MediaType, ParseParams}; -use deno_core::{ModuleCodeString, ModuleName, SourceMapData}; +use deno_core::{Extension, ModuleCodeString, ModuleName, SourceMapData}; use deno_error::JsErrorBox; -use deno_fetch::FetchPermissions; -use deno_net::NetPermissions; -use deno_web::{BlobStore, TimersPermission}; -use std::borrow::Cow; +use std::collections::HashSet; use std::env; +use std::fmt::Write as _; use std::io::Write; use std::path::{Path, PathBuf}; -use std::sync::Arc; - -pub struct PermissionsContainer; - -impl FetchPermissions for PermissionsContainer { - #[inline(always)] - fn check_net_url( - &mut self, - _url: &deno_core::url::Url, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } - - #[inline(always)] - fn check_read<'a>( - &mut self, - _path: Cow<'a, Path>, - _api_name: &str, - _get_path: &'a dyn deno_fs::GetPath, - ) -> Result, deno_io::fs::FsError> { - unreachable!("snapshotting") - } - - #[inline(always)] - fn check_write<'a>( - &mut self, - _path: Cow<'a, Path>, - _api_name: &str, - _get_path: &'a dyn deno_fs::GetPath, - ) -> Result, deno_io::fs::FsError> { - unreachable!("snapshotting") - } - - #[inline(always)] - fn check_net_vsock( - &mut self, - _cid: u32, - _port: u32, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } -} - -impl TimersPermission for PermissionsContainer { - #[inline(always)] - fn allow_hrtime(&mut self) -> bool { - true - } -} - -impl NetPermissions for PermissionsContainer { - fn check_read( - &mut self, - _p: &str, - _api_name: &str, - ) -> Result { - unreachable!("snapshotting") - } - - fn check_write( - &mut self, - _p: &str, - _api_name: &str, - ) -> Result { - unreachable!("snapshotting") - } - - fn check_net>( - &mut self, - _host: &(T, Option), - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } - - fn check_write_path<'a>( - &mut self, - _p: Cow<'a, Path>, - _api_name: &str, - ) -> Result, deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } - - fn check_vsock( - &mut self, - _cid: u32, - _port: u32, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - unreachable!("snapshotting") - } -} deno_core::extension!( fetch, @@ -109,38 +13,29 @@ deno_core::extension!( esm = ["src/runtime.js"], ); -// `extension_transpiler` callback for `deno_core::snapshot::create_snapshot`. +// `extension_transpiler` callback for `deno_core::snapshot::create_snapshot`, +// also applied to the residual lazy sources below. // -// Specialized to our snapshot's inputs. Of the eight deno_* extensions -// we register via `init()`, seven ship pre-built `.js` files -// in their `esm` lists (webidl/url/console/web/crypto/fetch/net) — only -// `deno_telemetry`'s `extension!` macro lists `.ts` files -// (`telemetry.ts`, `util.ts`), so the TypeScript branch is needed -// solely for that crate. Our local `fetch` extension contributes -// `src/runtime.js` (pure JS). No `node:` imports happen at snapshot -// build time, no `.mjs`, no user-supplied modules. So: +// Specialized to our inputs: the deno_* extensions ship `.js`, except +// `deno_telemetry` whose lazy scripts are `.ts`. Our local `fetch` extension +// contributes `src/runtime.js`. No `node:` imports, no `.mjs`, no +// user-supplied modules. So: // - `.js` → pass through. // - `.ts` → transpile via deno_ast (deno_telemetry only). // - anything else → build bug (deno shipping an unexpected file type // or us mislabelling one), panic loudly rather than emit a broken // snapshot. // -// No source maps: the snapshot is a binary blob the runtime loads — source -// maps would never be consumed. +// No source maps: nothing consumes them. // -// The signature still returns `Result<_, JsErrorBox>` because that's what -// `extension_transpiler` expects, but we never construct one — parse and -// transpile failures are build-time bugs in deno's own .ts internals (or -// in our runtime.js, if we ever change its extension), so they panic. +// The signature returns `Result<_, JsErrorBox>` because that's what +// `extension_transpiler` expects, but we never construct one: parse and +// transpile failures are build-time bugs, so they panic. // -// This replaces a call to `deno_runtime::transpile::maybe_transpile_source` -// from `deno_runtime 0.198.0`. The original is more general (handles -// `node:` modules, `.mjs`, emits source maps in debug builds, plumbs -// errors via `JsErrorBox`); none of that surface is reachable in our -// build. Dropping the `deno_runtime` dep eliminates a -// `deno_cache → rusqlite → libsqlite3-sys 0.35` transitive chain that -// collides with sqlx-sqlite's `libsqlite3-sys 0.30` (cargo's -// `links = "sqlite3"` rule). +// `deno_runtime::transpile::maybe_transpile_source` is the general version; +// depending on `deno_runtime` would pull a +// `deno_cache → rusqlite → libsqlite3-sys` chain that collides with +// sqlx-sqlite's `libsqlite3-sys` (cargo's `links = "sqlite3"` rule). fn maybe_transpile_source( name: ModuleName, source: ModuleCodeString, @@ -154,7 +49,7 @@ fn maybe_transpile_source( let parsed = deno_ast::parse_module(ParseParams { specifier: deno_core::url::Url::parse(&name).unwrap(), - text: source.into(), + text: source.as_str().into(), media_type, capture_tokens: false, scope_analysis: false, @@ -169,7 +64,13 @@ fn maybe_transpile_source( ..Default::default() }, &deno_ast::TranspileModuleOptions::default(), - &deno_ast::EmitOptions::default(), + // The default appends an inline source map comment, which is a + // syntax error where deno_core evaluates a lazy script as an + // expression. + &deno_ast::EmitOptions { + source_map: deno_ast::SourceMapOption::None, + ..Default::default() + }, ) .unwrap_or_else(|e| panic!("snapshot transpile: emit failed for {name}: {e}")) .into_source(); @@ -177,27 +78,66 @@ fn maybe_transpile_source( Ok((transpiled.text.into(), None)) } +/// One `(specifier, source)` table of the extension scripts the snapshot did +/// not evaluate. The snapshot only carries scripts that ran while it was built, +/// so a script first loaded in a live isolate has to come from here. +/// +/// deno_core binary-searches the table and reads each source as ASCII without +/// checking, so it must be sorted by specifier, free of duplicates, and ASCII. +fn residual_table( + name: &str, + files: impl Iterator, + consumed: &HashSet, +) -> String { + let mut files: Vec<_> = files.filter(|f| !consumed.contains(f.specifier)).collect(); + files.sort_by_key(|f| f.specifier); + files.dedup_by_key(|f| f.specifier); + + let mut out = format!("pub static {name}: &[(&str, &str)] = &[\n"); + for file in files { + let source = file + .load() + .unwrap_or_else(|e| panic!("could not load {}: {e}", file.specifier)); + let (code, _) = + maybe_transpile_source(ModuleName::from_static(file.specifier), source).unwrap(); + assert!( + code.as_str().is_ascii(), + "{} is not ASCII after transpiling", + file.specifier + ); + writeln!(out, " ({:?}, {:?}),", file.specifier, code.as_str()).unwrap(); + } + out.push_str("];\n"); + out +} + fn main() { println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); - let exts = vec![ - deno_telemetry::deno_telemetry::init(), - deno_webidl::deno_webidl::init(), - deno_url::deno_url::init(), - deno_console::deno_console::init(), - deno_web::deno_web::init::(Arc::new(BlobStore::default()), None), - deno_crypto::deno_crypto::init(None), - deno_fetch::deno_fetch::init::(Default::default()), - deno_net::deno_net::init::(None, None), + // Same list and order as `create_nativets_runtime`, which supplies their + // state through `lazy_init_extensions`. + let exts: Vec = vec![ + deno_telemetry::deno_telemetry::lazy_init(), + deno_webidl::deno_webidl::lazy_init(), + deno_web::deno_web::lazy_init(), + deno_crypto::deno_crypto::lazy_init(), + deno_fetch::deno_fetch::lazy_init(), + deno_net::deno_net::lazy_init(), fetch::init(), ]; - // Build the file path to the snapshot. + let lazy_js: Vec<_> = exts + .iter() + .flat_map(|e| e.lazy_loaded_js_files.iter().cloned()) + .collect(); + let lazy_esm: Vec<_> = exts + .iter() + .flat_map(|e| e.lazy_loaded_esm_files.iter().cloned()) + .collect(); + let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - let snapshot_path = o.join("FETCH_SNAPSHOT.bin"); - // Create the snapshot. let output = deno_core::snapshot::create_snapshot( deno_core::snapshot::CreateSnapshotOptions { cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"), @@ -213,9 +153,17 @@ fn main() { ) .unwrap(); - let mut file = std::fs::File::create(snapshot_path).unwrap(); + let mut file = std::fs::File::create(o.join("FETCH_SNAPSHOT.bin")).unwrap(); file.write_all(&output.output).unwrap(); + let consumed: HashSet = output.consumed_lazy_specifiers.into_iter().collect(); + let residual = format!( + "{}{}", + residual_table("RESIDUAL_LAZY_JS", lazy_js.into_iter(), &consumed), + residual_table("RESIDUAL_LAZY_ESM", lazy_esm.into_iter(), &consumed), + ); + std::fs::write(o.join("residual_lazy_sources.rs"), residual).unwrap(); + for path in output.files_loaded_during_snapshot { println!("cargo:rerun-if-changed={}", path.display()); } diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs index 75aacbcc64..750cdaa609 100644 --- a/backend/windmill-runtime-nativets/src/lib.rs +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -22,9 +22,7 @@ mod smoke_tests; mod cert_tests; use std::{ - borrow::Cow, cell::RefCell, - path::PathBuf, rc::Rc, sync::{Arc, LazyLock, Mutex}, }; @@ -32,17 +30,55 @@ use std::{ // Re-export deno_telemetry for use by windmill-worker's otel proxy pub use deno_telemetry; +/// The real environment and filesystem, except that the OTLP protocol always +/// reads as HTTP. +/// +/// The caller points `OTEL_EXPORTER_OTLP_ENDPOINT` at its own HTTP collector +/// only while [`init_telemetry`] runs, and the instance's own exporter shares +/// `OTEL_EXPORTER_OTLP_PROTOCOL`. Honouring `grpc` there would build an exporter +/// that reads the endpoint again on every export, and `console` one that never +/// reads it, so either would send job telemetry past that collector. +struct HttpOtlpSys; + +impl sys_traits::BaseEnvVar for HttpOtlpSys { + fn base_env_var_os(&self, key: &std::ffi::OsStr) -> Option { + if key == "OTEL_EXPORTER_OTLP_PROTOCOL" { + return Some("http/protobuf".into()); + } + sys_traits::impls::RealSys.base_env_var_os(key) + } +} + +impl sys_traits::BaseFsRead for HttpOtlpSys { + fn base_fs_read( + &self, + path: &std::path::Path, + ) -> std::io::Result> { + sys_traits::impls::RealSys.base_fs_read(path) + } +} + +/// `deno_telemetry::init` for the isolates of this process. +pub fn init_telemetry( + rt_config: deno_telemetry::OtelRuntimeConfig, + config: deno_telemetry::OtelConfig, +) -> anyhow::Result<()> { + deno_telemetry::init(&HttpOtlpSys, rt_config, config) +} + use deno_ast::ParseParams; use deno_core::{ + error::CoreErrorKind, op2, serde_v8, url, v8::{self, IsolateHandle}, Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions, }; use deno_error::JsErrorBox; -use deno_fetch::FetchPermissions; -use deno_net::NetPermissions; +use deno_permissions::{ + Permissions, PermissionsContainer, PermissionsOptions, RuntimePermissionDescriptorParser, +}; use deno_tls::{rustls::pki_types::CertificateDer, rustls::RootCertStore, RootCertStoreProvider}; -use deno_web::{BlobStore, TimersPermission}; +use deno_web::{BlobStore, InMemoryBroadcastChannel}; use itertools::Itertools; use lazy_static::lazy_static; use regex::Regex; @@ -57,156 +93,55 @@ use windmill_common::worker::{write_file, Connection, WINDMILL_DIR}; // ── Snapshot-matched extensions ────────────────────────────────────── // -// `deno_core` 0.352 validates that the snapshot's extension list is a -// *prefix* of the runtime's extension list (snapshot does not need an -// exact match — runtime is allowed to add extensions at the tail, but -// must not reorder or omit any that the snapshot baked in). +// The snapshot (build.rs) is built from the deno_* extensions in their +// `lazy_init()` form followed by this local `fetch` ext. deno_core requires the +// runtime's extension list to start with that same list in that same order; it +// may append more, which is how the windmill `ext` carrying our own ops gets in. // -// Our snapshot (in build.rs) is the same eight deno_* extensions ending -// with this local `fetch` ext. The runtime adds one extra entry at the -// end — the windmill `ext` carrying our own ops — which is fine because -// it's after the snapshot prefix. -// -// This local `fetch` extension declaration must be present in both -// build.rs and lib.rs so the type passes through the `init()` macro. -// The ESM is already in the snapshot, so this `init()` call at runtime -// is a no-op for esm — the registration just records the ext. +// This declaration must exist in both build.rs and lib.rs. The ESM is already +// in the snapshot, so `init()` at runtime only records the extension. deno_core::extension!( fetch, esm_entry_point = "ext:fetch/src/runtime.js", esm = ["src/runtime.js"], ); -// ── Permission container ───────────────────────────────────────────── +include!(concat!(env!("OUT_DIR"), "/residual_lazy_sources.rs")); -/// `no_network` is what `//no_network` promises: no outbound traffic of any kind, -/// the Windmill API included (the client module reaches it through `fetch`). -/// Enforced here and by disabling the deno_net ops, never in JS: user code can -/// call `Deno.core.ops.*` directly and skip any JS-level guard. -pub struct PermissionsContainer { - pub no_network: bool, -} - -const NO_NETWORK_ANNOTATION: &str = "no_network"; +// ── Permissions ────────────────────────────────────────────────────── /// Every op deno_net registers, see `create_nativets_runtime`. Read from the /// extension itself so an upgrade that adds an op cannot leave it enabled. static NO_NETWORK_DISABLED_OPS: LazyLock> = LazyLock::new(|| { - deno_net::deno_net::init::(None, None) + deno_net::deno_net::lazy_init() .ops .iter() .map(|op| op.name) .collect() }); -impl PermissionsContainer { - fn deny_net(&self, target: &str) -> Result<(), deno_permissions::PermissionCheckError> { - if self.no_network { - Err(deno_permissions::PermissionDeniedError::Fatal { - access: format!( - "net access to {target} (the script is annotated //{NO_NETWORK_ANNOTATION})" - ), - } - .into()) - } else { - Ok(()) - } - } -} - -impl FetchPermissions for PermissionsContainer { - #[inline(always)] - fn check_net_url( - &mut self, - url: &deno_core::url::Url, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - self.deny_net(&format!("\"{}\"", url.host_str().unwrap_or_default())) - } - - #[inline(always)] - fn check_read<'a>( - &mut self, - path: Cow<'a, std::path::Path>, - _api_name: &str, - _get_path: &'a dyn deno_fs::GetPath, - ) -> Result, deno_io::fs::FsError> { - Ok(deno_fs::CheckedPath::Unresolved(path)) - } - - #[inline(always)] - fn check_write<'a>( - &mut self, - path: Cow<'a, std::path::Path>, - _api_name: &str, - _get_path: &'a dyn deno_fs::GetPath, - ) -> Result, deno_io::fs::FsError> { - Ok(deno_fs::CheckedPath::Unresolved(path)) - } - - #[inline(always)] - fn check_net_vsock( - &mut self, - cid: u32, - port: u32, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - self.deny_net(&format!("vsock {cid}:{port}")) - } -} - -impl TimersPermission for PermissionsContainer { - #[inline(always)] - fn allow_hrtime(&mut self) -> bool { - true - } -} - -impl NetPermissions for PermissionsContainer { - fn check_read( - &mut self, - p: &str, - _api_name: &str, - ) -> Result { - self.deny_net(&format!("unix socket \"{p}\""))?; - Ok(PathBuf::from(p)) - } - - fn check_write( - &mut self, - p: &str, - _api_name: &str, - ) -> Result { - self.deny_net(&format!("unix socket \"{p}\""))?; - Ok(PathBuf::from(p)) - } - - fn check_net>( - &mut self, - host: &(T, Option), - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - self.deny_net(&format!("\"{}\"", host.0.as_ref())) - } - - fn check_write_path<'a>( - &mut self, - p: Cow<'a, std::path::Path>, - _api_name: &str, - ) -> Result, deno_permissions::PermissionCheckError> { - self.deny_net(&format!("unix socket \"{}\"", p.display()))?; - Ok(p) - } - - fn check_vsock( - &mut self, - cid: u32, - port: u32, - _api_name: &str, - ) -> Result<(), deno_permissions::PermissionCheckError> { - self.deny_net(&format!("vsock {cid}:{port}")) - } +/// What an isolate may do: reach the network, unless `no_network`, and nothing +/// else (no filesystem, environment, subprocess or FFI). Nothing prompts. +/// +/// `no_network` is what `//no_network` promises: no outbound traffic of any +/// kind, the Windmill API included (the client module reaches it through +/// `fetch`). Enforced here and by disabling the deno_net ops, never in JS: user +/// code can call `Deno.core.ops.*` directly and skip any JS-level guard. +fn isolate_permissions(no_network: bool) -> anyhow::Result { + let parser = Arc::new(RuntimePermissionDescriptorParser::new( + sys_traits::impls::RealSys, + )); + // An empty list means "all hosts", for a grant and for a denial alike. + let options = PermissionsOptions { + allow_net: (!no_network).then(Vec::new), + deny_net: no_network.then(Vec::new), + prompt: false, + ..Default::default() + }; + let permissions = Permissions::from_options(parser.as_ref(), &options) + .map_err(|e| anyhow::anyhow!("invalid isolate permissions: {e}"))?; + Ok(PermissionsContainer::new(parser, permissions)) } // ── Types ──────────────────────────────────────────────────────────── @@ -228,7 +163,7 @@ pub struct NativeAnnotation { /// script; `None` leaves the default in force. pub fetch_response_timeout_secs: Option, /// `//no_network`: the isolate may open no connection at all, see - /// [`PermissionsContainer`]. + /// [`isolate_permissions`]. pub no_network: bool, } @@ -413,13 +348,10 @@ pub fn setup_deno_runtime() -> anyhow::Result<()> { // deno_fetch requires a TLS provider; install ring as default (idempotent). let _ = rustls::crypto::ring::default_provider().install_default(); - let unrecognized_v8_flags = deno_core::v8_set_flags(vec![ - "--stack-size=1024".to_string(), - "--no-harmony-import-assertions".to_string(), - ]) - .into_iter() - .skip(1) - .collect::>(); + let unrecognized_v8_flags = deno_core::v8_set_flags(vec!["--stack-size=1024".to_string()]) + .into_iter() + .skip(1) + .collect::>(); if !unrecognized_v8_flags.is_empty() { init_err = Some(format!( @@ -435,7 +367,7 @@ pub fn setup_deno_runtime() -> anyhow::Result<()> { // SIGSEGV in WasmCodePointerTable::AllocateUninitializedEntry() on x86_64 Linux. // See: https://github.com/denoland/deno_core/issues/952 let platform = deno_core::v8::new_unprotected_default_platform(0, false).make_shared(); - deno_core::JsRuntime::init_platform(Some(platform), false); + deno_core::JsRuntime::init_platform(Some(platform)); }); if let Some(msg) = init_err { @@ -666,17 +598,15 @@ pub(crate) fn create_nativets_runtime( ..Default::default() }; + // Same list and order as the snapshot (build.rs); their state follows in + // `lazy_init_extensions` below, which must match it too. let exts: Vec = vec![ - deno_telemetry::deno_telemetry::init(), - deno_webidl::deno_webidl::init(), - deno_url::deno_url::init(), - deno_console::deno_console::init(), - deno_web::deno_web::init::(Arc::new(BlobStore::default()), None), - // Registered after deno_web to keep the snapshot (build.rs) a prefix of - // the runtime extension list; deno_crypto declares deps = [deno_webidl, deno_web]. - deno_crypto::deno_crypto::init(None), - deno_fetch::deno_fetch::init::(fetch_options), - deno_net::deno_net::init::(None, None), + deno_telemetry::deno_telemetry::lazy_init(), + deno_webidl::deno_webidl::lazy_init(), + deno_web::deno_web::lazy_init(), + deno_crypto::deno_crypto::lazy_init(), + deno_fetch::deno_fetch::lazy_init(), + deno_net::deno_net::lazy_init(), fetch::init(), ext, ]; @@ -688,6 +618,8 @@ pub(crate) fn create_nativets_runtime( deno_core::v8::CreateParams::default().heap_limits(0, 1024 * 1024 * 128), ), startup_snapshot: Some(RUNTIME_SNAPSHOT), + residual_lazy_js_sources: RESIDUAL_LAZY_JS, + residual_lazy_esm_sources: RESIDUAL_LAZY_ESM, module_loader: Some(Rc::new(deno_core::FsModuleLoader)), extension_transpiler: None, ..Default::default() @@ -714,12 +646,28 @@ pub(crate) fn create_nativets_runtime( y * 2 }); + js_runtime + .lazy_init_extensions(vec![ + deno_telemetry::deno_telemetry::args(), + deno_webidl::deno_webidl::args(), + deno_web::deno_web::args( + Arc::new(BlobStore::default()), + None, + false, + InMemoryBroadcastChannel::default(), + ), + deno_crypto::deno_crypto::args(None), + deno_fetch::deno_fetch::args(fetch_options), + deno_net::deno_net::args(None, None), + ]) + .map_err(windmill_common::error::to_anyhow)?; + let (log_sender, log_receiver) = mpsc::unbounded_channel::(); { let op_state = js_runtime.op_state(); let mut op_state = op_state.borrow_mut(); - op_state.put(PermissionsContainer { no_network }); + op_state.put(isolate_permissions(no_network)?); op_state.put(MainArgs { args: initial_args }); op_state.put(LogString { s: log_sender }); } @@ -731,7 +679,7 @@ pub(crate) fn create_nativets_runtime( .execute_script( "", format!( - "globalThis.__wmInitPerIsolate({{ fetchResponseTimeoutMs: {fetch_response_timeout_ms} }})" + "globalThis.__wmInitPerIsolate({{ fetchResponseTimeoutMs: {fetch_response_timeout_ms}, noNetwork: {no_network} }})" ), ) .map_err(windmill_common::error::to_anyhow)?; @@ -777,7 +725,7 @@ pub(crate) fn extract_global_string( js_runtime: &mut JsRuntime, global: v8::Global, ) -> Result, String> { - let scope = &mut js_runtime.handle_scope(); + deno_core::scope!(scope, js_runtime); let local = v8::Local::new(scope, global); match serde_v8::from_v8::>(scope, local) { Ok(s) => Ok(unsafe_raw(s.unwrap_or_else(|| "null".to_string()))), @@ -1169,7 +1117,10 @@ import("file:///eval.ts").then((module) => module.{main_fn}(...args)) Ok(global) => { extract_global_string(js_runtime, global).map_err(|e| ExecuteError::Script(e)) } - Err(deno_core::error::CoreError::Js(e)) => { + Err(e) if matches!(e.as_kind(), CoreErrorKind::Js(_)) => { + let CoreErrorKind::Js(e) = e.into_kind() else { + unreachable!("matched as a JS error above") + }; let source = e.frames.first().and_then(|f| { f.file_name .as_ref() diff --git a/backend/windmill-runtime-nativets/src/runtime.js b/backend/windmill-runtime-nativets/src/runtime.js index 88e10f765a..86d20dd022 100644 --- a/backend/windmill-runtime-nativets/src/runtime.js +++ b/backend/windmill-runtime-nativets/src/runtime.js @@ -1,39 +1,42 @@ -import * as abortSignal from "ext:deno_web/03_abort_signal.js"; -import * as domException from "ext:deno_web/01_dom_exception.js"; -import * as base64 from "ext:deno_web/05_base64.js"; -import * as console from "ext:deno_console/01_console.js"; -import * as encoding from "ext:deno_web/08_text_encoding.js"; -import * as event from "ext:deno_web/02_event.js"; -import * as fetch from "ext:deno_fetch/26_fetch.js"; -import * as file from "ext:deno_web/09_file.js"; -import * as fileReader from "ext:deno_web/10_filereader.js"; -import * as formData from "ext:deno_fetch/21_formdata.js"; -import * as headers from "ext:deno_fetch/20_headers.js"; -import * as streams from "ext:deno_web/06_streams.js"; -import * as timers from "ext:deno_web/02_timers.js"; -import * as url from "ext:deno_url/00_url.js"; -import * as net from "ext:deno_net/01_net.js"; -import * as tls from "ext:deno_net/02_tls.js"; -import * as urlPattern from "ext:deno_url/01_urlpattern.js"; -import * as webidl from "ext:deno_webidl/00_webidl.js"; -import * as crypto from "ext:deno_crypto/00_crypto.js"; -import * as response from "ext:deno_fetch/23_response.js"; -import * as request from "ext:deno_fetch/23_request.js"; -import "ext:deno_web/02_structured_clone.js"; -import * as globalInterfaces from "ext:deno_web/04_global_interfaces.js"; -// Namespace imports (not side-effect-only) so their constructors are reachable -// for the globalThis wiring below. The module bodies still execute on -// evaluation, so their side effects apply. -import * as messagePort from "ext:deno_web/13_message_port.js"; -import * as compression from "ext:deno_web/14_compression.js"; -import * as performance from "ext:deno_web/15_performance.js"; -import "ext:deno_web/16_image_data.js"; -import "ext:deno_fetch/27_eventsource.js"; +import { core, internals } from "ext:core/mod.js"; + +// deno_crypto brands CryptoKeys with this symbol and reads it from `internals` +// when it evaluates; in Deno a deno_node module sets it, which is not loaded here. +internals.kKeyObject ??= Symbol("kKeyObject"); + +// Every script loaded here runs at snapshot-build time and is baked into the +// snapshot. One that is first loaded at run time comes from the residual table +// build.rs emits instead. +const abortSignal = core.loadExtScript("ext:deno_web/03_abort_signal.js"); +const domException = core.loadExtScript("ext:deno_web/01_dom_exception.js"); +const base64 = core.loadExtScript("ext:deno_web/05_base64.js"); +const console = core.loadExtScript("ext:deno_web/01_console.js"); +const encoding = core.loadExtScript("ext:deno_web/08_text_encoding.js"); +const event = core.loadExtScript("ext:deno_web/02_event.js"); +const fetch = core.loadExtScript("ext:deno_fetch/26_fetch.js"); +const file = core.loadExtScript("ext:deno_web/09_file.js"); +const fileReader = core.loadExtScript("ext:deno_web/10_filereader.js"); +const formData = core.loadExtScript("ext:deno_fetch/21_formdata.js"); +const headers = core.loadExtScript("ext:deno_fetch/20_headers.js"); +const streams = core.loadExtScript("ext:deno_web/06_streams.js"); +const timers = core.loadExtScript("ext:deno_web/02_timers.js"); +const url = core.loadExtScript("ext:deno_web/00_url.js"); +const urlPattern = core.loadExtScript("ext:deno_web/01_urlpattern.js"); +const webidl = core.loadExtScript("ext:deno_webidl/00_webidl.js"); +const crypto = core.loadExtScript("ext:deno_crypto/00_crypto.js"); +const response = core.loadExtScript("ext:deno_fetch/23_response.js"); +const request = core.loadExtScript("ext:deno_fetch/23_request.js"); +const globalInterfaces = core.loadExtScript( + "ext:deno_web/04_global_interfaces.js", +); +const messagePort = core.loadExtScript("ext:deno_web/13_message_port.js"); +const compression = core.loadExtScript("ext:deno_web/14_compression.js"); +const performance = core.loadExtScript("ext:deno_web/15_performance.js"); // deno_fetch applies no deadline, so a peer that accepts a request and then // never answers leaves `await fetch(...)` pending until the job timeout, which // self-hosted defaults to 7 days. -const ORIGINAL_FETCH = fetch.fetch; +const DENO_FETCH = fetch.fetch; // deno_web's timers reject any `this` other than undefined/globalThis, so // `timers.setTimeout(...)` passes the module namespace and throws "Illegal @@ -50,6 +53,14 @@ const abortControllerAbort = Function.prototype.call.bind( ); const ReflectApply = Reflect.apply; +// Forwards the original argument count, like every caller below relies on. The +// extra promise hop under `no_network` costs the same-tick settling of an +// aborted fetch, which only matters where a request could have been made. +function ORIGINAL_FETCH() { + const p = ReflectApply(DENO_FETCH, undefined, arguments); + return noNetwork ? promiseThen(p, undefined, rewordNoNetworkDenial) : p; +} + // Installed per isolate by __wmInitPerIsolate; 0 disables. Only a backstop // for the impossible case of fetch running before that init. let fetchResponseTimeoutMs = 900_000; @@ -74,6 +85,21 @@ function fetchResponseTimeoutError(requestUrl, timeoutMs) { ); } +// Set per isolate for a `//no_network` script. The denial itself is the +// isolate's permissions (Rust); this only replaces deno's "run again with the +// --allow-net flag" hint, which names a flag a script author cannot pass. +let noNetwork = false; + +function rewordNoNetworkDenial(e) { + if (e?.name === "NotCapable" && typeof e.message === "string") { + e.message = e.message.replace( + ", run again with the --allow-net flag", + ": the script is annotated //no_network", + ); + } + throw e; +} + globalThis.atob = base64.atob; globalThis.btoa = base64.btoa; // Not `async`, for the same reason deno_fetch's own outer fetch isn't: WPT @@ -152,7 +178,22 @@ globalThis.console = new console.Console((msg, level) => ); globalThis.AbortController = abortSignal.AbortController; globalThis.AbortSignal = abortSignal.AbortSignal; -globalThis.crypto = crypto.crypto; +// A getter, not a value: `crypto.crypto` mints a cppgc object, and the cppgc +// heap is not attached while this module runs at snapshot-build time. +Object.defineProperty(globalThis, "crypto", { + configurable: true, + enumerable: true, + get: () => crypto.crypto, + // Assignment has to keep working, as it does on a plain property. + set(value) { + Object.defineProperty(globalThis, "crypto", { + configurable: true, + enumerable: true, + writable: true, + value, + }); + }, +}); globalThis.Crypto = crypto.Crypto; globalThis.CryptoKey = crypto.CryptoKey; globalThis.SubtleCrypto = crypto.SubtleCrypto; @@ -233,6 +274,7 @@ globalThis.__wmInitPerIsolate = (config) => { if (config != null && typeof config.fetchResponseTimeoutMs === "number") { fetchResponseTimeoutMs = config.fetchResponseTimeoutMs; } + noNetwork = config?.noNetwork === true; // setTimeOrigin() seeds performance.timeOrigin from the isolate's wall clock; // without it timeOrigin is undefined and `timeOrigin + performance.now()` is NaN. @@ -256,13 +298,14 @@ globalThis.__wmInitPerIsolate = (config) => { }; // Expose bootstrapOtel globally so it can be called from Rust after runtime creation. -// We use dynamic import so deno_telemetry isn't loaded during snapshot creation. +// Loaded on first call so deno_telemetry isn't evaluated during snapshot creation. // Config: [tracingEnabled, metricsEnabled, consoleConfig, deterministic] // consoleConfig: 0=ignore, 1=capture, 2=replace globalThis.__bootstrapOtel = () => { - import("ext:deno_telemetry/telemetry.ts").then(({ bootstrap, enterSpan }) => { - bootstrap([1, 0, 1, 0]); - // Expose enterSpan for setting parent trace context - globalThis.__enterSpan = enterSpan; - }); + const { bootstrap, enterSpan } = core.loadExtScript( + "ext:deno_telemetry/telemetry.ts", + ); + bootstrap([1, 0, 1, 0]); + // Expose enterSpan for setting parent trace context + globalThis.__enterSpan = enterSpan; }; diff --git a/backend/windmill-runtime-nativets/src/smoke_tests.rs b/backend/windmill-runtime-nativets/src/smoke_tests.rs index f3da460227..1373a8abd4 100644 --- a/backend/windmill-runtime-nativets/src/smoke_tests.rs +++ b/backend/windmill-runtime-nativets/src/smoke_tests.rs @@ -451,6 +451,10 @@ export async function main(): Promise<{ uuid: string; nonzero: boolean; sha256: .map((b) => b.toString(16).padStart(2, "0")) .join(""); + // `crypto` is installed as an accessor; polyfill-style assignment must + // still work rather than throw for want of a setter. + (globalThis as any).crypto = globalThis.crypto; + return { uuid, nonzero, sha256 }; } "#; diff --git a/backend/windmill-runtime-nativets/tests/no_network.rs b/backend/windmill-runtime-nativets/tests/no_network.rs index 876207fc26..dc7086f22e 100644 --- a/backend/windmill-runtime-nativets/tests/no_network.rs +++ b/backend/windmill-runtime-nativets/tests/no_network.rs @@ -55,6 +55,9 @@ async function attempt(f) {{ export async function main() {{ return {{ fetch: await attempt(() => fetch("http://127.0.0.1:{port}/").then((r) => r.text())), + data_url: await attempt(() => fetch("data:text/plain,ok").then((r) => r.text())), + blob_url: await attempt(() => fetch(URL.createObjectURL(new Blob(["ok"]))).then((r) => r.text())), + raw_fetch: await attempt(() => Deno.core.ops.op_fetch("GET", "http://127.0.0.1:{port}/", [], null, false, null, null)), tcp: await attempt(() => Deno.core.ops.op_net_connect_tcp({{ hostname: "127.0.0.1", port: {port} }}, null, null)), unix: await attempt(() => Deno.core.ops.op_net_connect_unix("/var/run/docker.sock")), quic: await attempt(() => Deno.core.ops.op_quic_endpoint_create({{ hostname: "localhost", port: 0 }}, true)), @@ -76,6 +79,15 @@ async fn no_network_denies_fetch_and_raw_socket_ops() { fetch.contains("no_network"), "fetch was not denied: {fetch}" ); + // Neither opens a connection, so neither is network access. + assert_eq!(out["data_url"], "allowed", "{out}"); + assert_eq!(out["blob_url"], "allowed", "{out}"); + // The global `fetch` only words the refusal; the op under it is what refuses. + let raw_fetch = out["raw_fetch"].as_str().unwrap(); + assert!( + raw_fetch.contains("Requires net access"), + "the fetch op was not denied: {raw_fetch}" + ); for key in ["tcp", "unix", "quic"] { let msg = out[key].as_str().unwrap(); assert!( diff --git a/backend/windmill-runtime-nativets/tests/otel_e2e.rs b/backend/windmill-runtime-nativets/tests/otel_e2e.rs index 22fd18de90..d1853c461d 100644 --- a/backend/windmill-runtime-nativets/tests/otel_e2e.rs +++ b/backend/windmill-runtime-nativets/tests/otel_e2e.rs @@ -71,7 +71,7 @@ async fn spawn_capturing_http(captured: Arc>>>) -> u16 { /// drifts away from `tracing_enabled + Capture`, this test breaks /// before the customer-facing panic does. fn init_with_ee_otel_config() { - deno_telemetry::init( + windmill_runtime_nativets::init_telemetry( deno_telemetry::OtelRuntimeConfig { runtime_name: "windmill-nativets".into(), runtime_version: "0".into(), @@ -157,13 +157,12 @@ export async function main(): Promise {{ "fetch target should have been hit exactly once" ); - // 4. Force the BatchSpanProcessor to flush so the exporter posts - // to our mock collector synchronously (default flush interval - // is ~5s; tests can't wait that long). - deno_telemetry::flush(); - // Exporter is async over the OTel runtime; give it a beat to - // actually send the HTTP request. - tokio::time::sleep(Duration::from_millis(500)).await; + // 4. The BatchSpanProcessor exports on its own schedule (about 5s by + // default) and deno_telemetry exposes no flush, so wait for it. + let deadline = tokio::time::Instant::now() + Duration::from_secs(20); + while otlp_hits.lock().await.is_empty() && tokio::time::Instant::now() < deadline { + tokio::time::sleep(Duration::from_millis(100)).await; + } let otlp_captured = otlp_hits.lock().await; assert!( diff --git a/backend/windmill-runtime-nativets/tests/otel_init.rs b/backend/windmill-runtime-nativets/tests/otel_init.rs index fdf1ac944d..80ef8683aa 100644 --- a/backend/windmill-runtime-nativets/tests/otel_init.rs +++ b/backend/windmill-runtime-nativets/tests/otel_init.rs @@ -46,7 +46,7 @@ async fn ee_otel_init_config_populates_otel_globals() { // Step 1: reproduce the footgun. `OtelConfig::default()` has // tracing/metrics off and `console = Ignore`, which trips the // 0.31 early-return — Ok(()) but OnceCell stays empty. - deno_telemetry::init( + windmill_runtime_nativets::init_telemetry( deno_telemetry::OtelRuntimeConfig { runtime_name: "windmill-nativets-test".into(), runtime_version: "0".into(), @@ -63,10 +63,17 @@ async fn ee_otel_init_config_populates_otel_globals() { tracing_enabled/console fields." ); + // The instance's own exporter shares this variable and may be set to gRPC, + // or to anything else. Native telemetry must not read it: a value + // deno_telemetry rejects outright shows it is never consulted. + // SAFETY: the only test in this binary, so nothing reads the environment + // concurrently. + unsafe { std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", "not-a-protocol") }; + // Step 2: re-call init with the exact config shape that the EE // `load_internal_otel_exporter` ships — this is what production // hits, so the test exercises the actual prod call path. - deno_telemetry::init( + windmill_runtime_nativets::init_telemetry( deno_telemetry::OtelRuntimeConfig { runtime_name: "windmill-nativets".into(), runtime_version: "0".into(), diff --git a/backend/windmill-worker/src/ai/stream_event_processor.rs b/backend/windmill-worker/src/ai/stream_event_processor.rs index fe8a3b9b00..c4d17f080a 100644 --- a/backend/windmill-worker/src/ai/stream_event_processor.rs +++ b/backend/windmill-worker/src/ai/stream_event_processor.rs @@ -1,3 +1,8 @@ +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + use async_trait::async_trait; use windmill_ai::{query_builder::StreamEventSink, types::*}; use windmill_common::{error::Error, worker::Connection}; @@ -10,11 +15,13 @@ use crate::job_logger::append_result_stream; pub struct StreamEventProcessor { tx: Option>, pub handle: Option>, + /// Shared by every clone, so a sink lent to a parser counts into its owner. + streamed: Arc, } impl Clone for StreamEventProcessor { fn clone(&self) -> Self { - Self { tx: self.tx.clone(), handle: None } + Self { tx: self.tx.clone(), handle: None, streamed: self.streamed.clone() } } } @@ -48,13 +55,19 @@ impl StreamEventProcessor { } }); - Self { tx: Some(tx), handle: Some(handle) } + Self { tx: Some(tx), handle: Some(handle), streamed: Arc::default() } } /// Create a silent StreamEventProcessor that only accumulates events locally /// without persisting them to the database. Used when streaming is disabled. pub fn new_silent() -> Self { - Self { tx: None, handle: None } + Self { tx: None, handle: None, streamed: Arc::default() } + } + + /// How many events have been streamed to the job's readers. A silent processor + /// streams none. + pub fn streamed_events(&self) -> usize { + self.streamed.load(Ordering::Relaxed) } pub fn to_handle(self) -> Option> { @@ -79,6 +92,7 @@ impl StreamEventSink for StreamEventProcessor { Ok(event_json) => { let event_json = format!("{}\n", event_json); events_str.push_str(&event_json); + self.streamed.fetch_add(1, Ordering::Relaxed); if let Err(err) = tx .send(event_json.clone()) diff --git a/backend/windmill-worker/src/ai_executor.rs b/backend/windmill-worker/src/ai_executor.rs index 7dd2382731..1b889aea34 100644 --- a/backend/windmill-worker/src/ai_executor.rs +++ b/backend/windmill-worker/src/ai_executor.rs @@ -43,6 +43,7 @@ use windmill_ai::{ common_outbound_headers, needs_unavailable_oauth_exchange, retain_effective_credentials, }, query_builder::{BuildRequestArgs, ParsedResponse}, + retry::{as_transient, into_final_error, response_error, send_error, Backoff, MAX_RETRIES}, types::*, utils::{pinned_ai_client_for, should_use_structured_output_tool}, }; @@ -1659,10 +1660,17 @@ pub async fn run_agent( .await; let mut retried_context = false; + let mut backoff = Backoff::default(); + let streamed_events = || { + stream_event_processor + .as_ref() + .map_or(0, |processor| processor.streamed_events()) + }; let (parsed, request_message_count) = loop { // How many messages this request carries, so the provider's prompt count can // later be told apart from what the response and its tool results add. let request_message_count = messages.context().len(); + let streamed_before = streamed_events(); // Handle AWS Bedrock provider specially using the official SDK let attempt: error::Result<_> = async { @@ -1818,7 +1826,7 @@ pub async fn run_agent( let resp = build_http_request(&endpoint, &auth_headers, request_body) .send() .await - .map_err(|e| Error::internal_err(format!("Failed to call API: {}", e)))?; + .map_err(|e| send_error(e, "Failed to call API"))?; match resp.error_for_status_ref() { Ok(_) => { @@ -1829,6 +1837,7 @@ pub async fn run_agent( } Err(e) => { let status = resp.status(); + let headers = resp.headers().clone(); let text = resp .text() .await @@ -1898,10 +1907,9 @@ pub async fn run_agent( query_builder = create_chat_completions_query_builder(&credentials); include_usage = true; } else { - return Err(Error::internal_err(format!( - "API error calling {}: {} - {}", - endpoint, e, text - ))); + let message = + format!("API error calling {}: {} - {}", endpoint, e, text); + return Err(response_error(status, &headers, &text, message)); } } } @@ -1961,7 +1969,45 @@ pub async fn run_agent( return Err(error); } } - Err(error) => return Err(error), + Err(error) => { + let Some(transient) = as_transient(&error) else { + return Err(error); + }; + // Readers have no way to take back what they were shown, so a call + // that already streamed part of its answer fails as it is. + if streamed_events() != streamed_before { + append_logs( + &job.id, + &job.workspace_id, + "AI provider call failed after part of its answer had streamed; not retried.\n".to_string(), + conn, + ) + .await; + return Err(into_final_error(error)); + } + let Some(delay) = backoff.next_delay(transient) else { + return Err(into_final_error(error)); + }; + append_logs( + &job.id, + &job.workspace_id, + format!( + "AI provider call failed: {}. Retrying in {:.1}s (retry {}/{MAX_RETRIES}).\n", + transient.message, + delay.as_secs_f64(), + backoff.retries() + ), + conn, + ) + .await; + let mut cancelled = cancel_rx.clone(); + tokio::select! { + _ = tokio::time::sleep(delay) => {} + Ok(_) = cancelled.wait_for(|cancelled| *cancelled) => { + return Err(Error::ExecutionErr("Job cancelled".to_string())); + } + } + } } }; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index bb47d3e4be..de33a2b9e6 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -3273,9 +3273,9 @@ pub async fn handle_wac_v2_output( .or(if own_runnable { None } else { job.priority }), None, // authed false, // running - None, // end_user_email - None, // trigger - None, // suspended_mode + job.permissioned_as_end_user_email.clone(), + None, // trigger + None, // suspended_mode job.job_token_scopes.as_deref(), ) .await?; @@ -3731,7 +3731,10 @@ pub async fn handle_wac_v2_output( // status row, the order every child completion takes. let segment_ms = sqlx::query_scalar!( "WITH prev AS (SELECT started_at FROM v2_job_queue WHERE id = $1) - UPDATE v2_job_queue q SET running = false, started_at = null + UPDATE v2_job_queue q SET running = false, started_at = null, + extras = CASE WHEN q.started_at IS NULL OR q.extras ? 'wac_first_started_at' THEN q.extras + ELSE jsonb_set(COALESCE(q.extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(q.started_at)) + END FROM prev WHERE q.id = $1 RETURNING (extract(epoch FROM now() - prev.started_at) * 1000)::bigint", job.id, @@ -4083,6 +4086,7 @@ async fn handle_dedicated_bunnative( result, result_columns: None, mem_peak: 0, + resource_usage: None, canceled_by: None, success, cached_res_path: None, diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 745f202826..6f6c48baec 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1962,6 +1962,18 @@ pub async fn save_in_cache( CACHED_RESULTS.insert(cached_path, Arc::new(store_cache_resource)); } +pub(crate) fn deno_not_installed_error() -> Error { + Error::ExecutionErr(format!( + "Deno is not installed on this worker (looked for it at {}). The standard Windmill images \ + no longer ship Deno: official support for it on those images was discontinued after Deno \ + announced the end of development of the Deno runtime (https://deno.com/blog/cloudflare). \ + We recommend migrating this script to Bun. To keep running Deno scripts, use the \ + windmill-full or windmill-ee-full image on every worker that accepts the deno tag, or \ + install Deno on them and point DENO_PATH at it.", + crate::DENO_PATH.as_str() + )) +} + fn tentatively_improve_error(err: Error, executable: &str) -> Error { #[cfg(unix)] let err_msgs = vec!["os error 2", "os error 3", "No such file or directory"]; @@ -1970,6 +1982,11 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error { let err_msgs = vec!["program not found", "os error 2", "os error 3"]; if err_msgs.iter().any(|msg| err.to_string().contains(msg)) { + // An absolute DENO_PATH is checked before spawning, where a missing job dir + // cannot be mistaken for a missing binary. + if executable == crate::DENO_PATH.as_str() && !Path::new(executable).is_absolute() { + return deno_not_installed_error(); + } return Error::internal_err(format!( "Executable {executable} not found on worker. PATH: {}", *PATH_ENV diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 2c5c17bb27..58388c3cfd 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -7,7 +7,8 @@ use windmill_queue::{append_logs, CanceledBy, MiniPulledJob}; use crate::{ common::{ - build_command_with_isolation, create_args_and_out_file, get_reserved_variables, + build_command_with_isolation, create_args_and_out_file, deno_not_installed_error, + get_reserved_variables, parse_npm_config, read_file, read_result, start_child_process, OccupancyMetrics, StreamNotifier, }, @@ -155,6 +156,17 @@ async fn get_common_deno_proc_envs( return deno_envs; } +/// A spawn failure only names Deno when Deno itself is the spawned executable; behind an +/// isolation wrapper or a dedicated worker it surfaces as an opaque exit. A bare `DENO_PATH` +/// is resolved through PATH at spawn time, so it is left to the spawn error. +fn ensure_deno_installed() -> Result<()> { + let path = std::path::Path::new(DENO_PATH.as_str()); + if path.is_absolute() && !path.exists() { + return Err(deno_not_installed_error()); + } + Ok(()) +} + pub async fn generate_deno_lock( job_id: &Uuid, code: &str, @@ -167,6 +179,7 @@ pub async fn generate_deno_lock( base_internal_url: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> error::Result { + ensure_deno_installed()?; let _ = write_file(job_dir, "main.ts", code)?; let import_map_path = format!("{job_dir}/import_map.json"); @@ -249,6 +262,7 @@ pub async fn handle_deno_job( occupancy_metrics: &mut OccupancyMetrics, has_stream: &mut bool, ) -> error::Result> { + ensure_deno_installed()?; let annotations = TypeScriptAnnotations::parse(inner_content); // let mut start = Instant::now(); @@ -745,6 +759,7 @@ pub async fn start_worker( use crate::common::build_envs_map; + ensure_deno_installed()?; let _ = write_file(job_dir, "main.ts", inner_content)?; let common_deno_proc_envs = get_common_deno_proc_envs( &token, diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index c25ad82a83..31eb8034f9 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -59,6 +59,56 @@ use windmill_common::tracing_init::{OTEL_JOB_LOGS, OTEL_PREFIX, QUIET_MODE, VERB lazy_static::lazy_static! { pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true"); + // A job runs several children in turn (install, build, run): their CPU time adds up + // here until the job completes. + static ref JOB_CPU_TIME_MS: std::sync::Mutex> = Default::default(); +} + +/// CPU time of the children run so far for the job, which stops being tracked. +pub(crate) fn take_job_cpu_time_ms(job_id: &Uuid) -> Option { + JOB_CPU_TIME_MS.lock().unwrap().remove(job_id) +} + +/// CPU time in ms from the content of `/proc//stat`: the process's own user and +/// system time plus that of the children it waited for, which is where the time of a +/// job run under a wrapper (nsjail, unshare) ends up. +#[cfg_attr(not(target_os = "linux"), allow(dead_code))] +fn parse_proc_stat_cpu_ms(stat: &str) -> Option { + // The values are in USER_HZ ticks, which Linux fixes at 100 per second. + const MS_PER_TICK: i64 = 10; + // `comm` is free text in parentheses: the numbered fields resume after its last `)`, + // with the state as field 3 and utime, stime, cutime, cstime as fields 14 to 17. + let fields: Vec<&str> = stat.rsplit_once(')')?.1.split_ascii_whitespace().collect(); + let ticks = fields + .get(11..15)? + .iter() + .map(|f| f.parse::().ok()) + .sum::>()?; + Some(ticks * MS_PER_TICK) +} + +/// Resolves once the child has exited, to its CPU time. The child is left un-reaped: +/// `/proc//stat` is only readable until then, so the caller reaps afterwards. +#[cfg(target_os = "linux")] +async fn wait_for_exit_cpu_time_ms(pid: u32) -> Option { + use nix::sys::wait::{waitid, Id, WaitPidFlag}; + let (tx, rx) = tokio::sync::oneshot::channel(); + // A plain thread rather than `spawn_blocking`: the wait cannot be cancelled, and a + // blocking task still parked on a live child would hold up the runtime's shutdown. + std::thread::Builder::new() + .name("job-cpu-time".to_string()) + .spawn(move || { + let cpu = waitid( + Id::Pid(Pid::from_raw(pid as i32)), + WaitPidFlag::WEXITED | WaitPidFlag::WNOWAIT, + ) + .ok() + .and_then(|_| std::fs::read_to_string(format!("/proc/{pid}/stat")).ok()) + .and_then(|stat| parse_proc_stat_cpu_ms(&stat)); + let _ = tx.send(cpu); + }) + .ok()?; + rx.await.ok().flatten() } // - kill windows process along with all child processes @@ -267,7 +317,15 @@ pub async fn handle_child( let wait_on_child = async { let kill_reason = tokio::select! { biased; - result = Box::into_pin(child.wait()) => return result.map(Ok), + result = async { + #[cfg(target_os = "linux")] + if let Some(pid) = pid.filter(|_| job_id != Uuid::nil()) { + if let Some(cpu) = wait_for_exit_cpu_time_ms(pid).await { + *JOB_CPU_TIME_MS.lock().unwrap().entry(job_id).or_insert(0) += cpu; + } + } + Box::into_pin(child.wait()).await + } => return result.map(Ok), Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs, _ = sleep(timeout_duration) => KillReason::Timeout { phase: child_name.to_string(), @@ -1049,6 +1107,7 @@ where let wm_memory_usage = get_windmill_memory_usage(); tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024); let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics()); + crate::resource_metrics::record_worker_resources(worker_name, memory_usage, wm_memory_usage, occupancy.as_ref()); if job_id != Uuid::nil() { if let Err(err) = update_worker_ping_from_job(&conn, &job_id, w_id, worker_name, memory_usage, wm_memory_usage, occupancy).await { tracing::error!("Unable to update worker ping for job {} in workspace {}. Error was: {:?}", job_id, w_id, err); @@ -1213,6 +1272,21 @@ pub fn process_status( mod tests { use super::*; + #[test] + fn proc_stat_cpu_time_survives_a_hostile_process_name() { + // utime 150, stime 50, cutime 700, cstime 100 ticks: 1000 ticks, 10 s. + let tail = "Z 1 42 42 0 -1 4194560 100 200 0 0 150 50 700 100 20 0 1 0 12345 0 0"; + assert_eq!( + parse_proc_stat_cpu_ms(&format!("42 (bun) {tail}")), + Some(10_000) + ); + assert_eq!( + parse_proc_stat_cpu_ms(&format!("42 (a) R 9 (9 9 9 9 9 9 9 9 9) {tail}")), + Some(10_000) + ); + assert_eq!(parse_proc_stat_cpu_ms("42 (bun) Z 1 42"), None); + } + // A non-positive timeout means "unset" to `resolve_job_timeout`, so an // exhausted budget that reported 0 would hand the next phase no deadline at // all — the opposite of what running out of time should do. diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 6059936af6..aa1d9ab209 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -81,6 +81,7 @@ mod prepare_deps; mod python_executor; #[cfg(feature = "python")] mod python_versions; +mod resource_metrics; pub mod result_processor; #[cfg(feature = "rust")] mod rust_executor; diff --git a/backend/windmill-worker/src/resource_metrics.rs b/backend/windmill-worker/src/resource_metrics.rs new file mode 100644 index 0000000000..53c7dc5b4d --- /dev/null +++ b/backend/windmill-worker/src/resource_metrics.rs @@ -0,0 +1,235 @@ +use std::sync::{atomic::Ordering, Mutex, OnceLock}; + +use windmill_common::{ + otel_oss::{ + otel_add_worker_cpu, otel_record_job_memory_peak, otel_set_worker_memory, + otel_set_worker_occupancy_rate, + }, + worker::{get_cpu_stat, get_memory, CpuStat, WORKER_GROUP}, + OTEL_METRICS_ENABLED, +}; + +use crate::common::OccupancyResult; + +#[cfg(feature = "prometheus")] +struct PromMetrics { + memory_usage: prometheus::IntGaugeVec, + windmill_memory_usage: prometheus::IntGaugeVec, + memory_limit: prometheus::IntGaugeVec, + cpu_usage: prometheus::CounterVec, + cpu_periods: prometheus::IntCounterVec, + cpu_throttled_periods: prometheus::IntCounterVec, + cpu_throttled_seconds: prometheus::CounterVec, + occupancy_rate: prometheus::GaugeVec, + job_memory_peak: prometheus::HistogramVec, +} + +#[cfg(feature = "prometheus")] +lazy_static::lazy_static! { + static ref PROM_METRICS: Option = + if windmill_common::METRICS_ENABLED.load(Ordering::Relaxed) { + const WORKER: &[&str] = &["name", "worker_group"]; + Some(PromMetrics { + memory_usage: prometheus::register_int_gauge_vec!( + "worker_memory_usage_bytes", + "Memory used by the worker's container, page cache excluded", + WORKER + ) + .unwrap(), + windmill_memory_usage: prometheus::register_int_gauge_vec!( + "worker_windmill_memory_usage_bytes", + "Resident memory of the worker process itself", + WORKER + ) + .unwrap(), + memory_limit: prometheus::register_int_gauge_vec!( + "worker_memory_limit_bytes", + "Memory limit of the worker's container", + WORKER + ) + .unwrap(), + cpu_usage: prometheus::register_counter_vec!( + "worker_cpu_usage_seconds_total", + "CPU time consumed by the worker's container", + WORKER + ) + .unwrap(), + cpu_periods: prometheus::register_int_counter_vec!( + "worker_cpu_periods_total", + "CPU enforcement periods elapsed for the worker's container", + WORKER + ) + .unwrap(), + cpu_throttled_periods: prometheus::register_int_counter_vec!( + "worker_cpu_throttled_periods_total", + "CPU enforcement periods in which the worker's container was throttled", + WORKER + ) + .unwrap(), + cpu_throttled_seconds: prometheus::register_counter_vec!( + "worker_cpu_throttled_seconds_total", + "Time the worker's container spent throttled by its CPU limit", + WORKER + ) + .unwrap(), + occupancy_rate: prometheus::register_gauge_vec!( + "worker_occupancy_rate", + "Share of time the worker spent executing jobs, over the window", + &["name", "worker_group", "window"] + ) + .unwrap(), + job_memory_peak: prometheus::register_histogram_vec!( + "worker_job_memory_peak_bytes", + "Peak memory of a job's main process", + &["tag"], + windmill_common::worker::JOB_MEMORY_PEAK_BUCKETS.to_vec() + ) + .unwrap(), + }) + } else { + None + }; +} + +fn metrics_enabled() -> bool { + #[cfg(feature = "prometheus")] + if PROM_METRICS.is_some() { + return true; + } + OTEL_METRICS_ENABLED.load(Ordering::Relaxed) +} + +// Memory and CPU are read from the cgroup, which every worker of a process shares +// (native mode runs several). Only the first worker to report exports them, so that +// summing the series over workers does not count one container several times. +static CONTAINER_SERIES_OWNER: OnceLock = OnceLock::new(); +static MEMORY_LIMIT: Mutex>> = Mutex::new(None); + +/// Replaces the exported memory limit, for the ping that re-reads the cgroup limits. +pub(crate) fn set_memory_limit(limit: Option) { + *MEMORY_LIMIT.lock().unwrap() = Some(limit); +} +static LAST_CPU_STAT: Mutex = + Mutex::new(CpuStat { usage_usec: 0, nr_periods: 0, nr_throttled: 0, throttled_usec: 0 }); + +/// Exports the worker's resource readings. Meant for the ping ticks, which have already +/// read memory usage and occupancy; the only extra work is the cgroup `cpu.stat` read. +pub(crate) fn record_worker_resources( + worker_name: &str, + memory_usage: Option, + wm_memory_usage: Option, + occupancy: Option<&OccupancyResult>, +) { + if !metrics_enabled() { + return; + } + let worker_group = WORKER_GROUP.as_str(); + + if let Some(occupancy) = occupancy { + let windows = [ + ("total", Some(occupancy.occupancy_rate)), + ("15s", occupancy.occupancy_rate_15s), + ("5m", occupancy.occupancy_rate_5m), + ("30m", occupancy.occupancy_rate_30m), + ]; + for (window, rate) in windows { + let Some(rate) = rate.filter(|r| r.is_finite()) else { + continue; + }; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + m.occupancy_rate + .with_label_values(&[worker_name, worker_group, window]) + .set(rate as f64); + } + otel_set_worker_occupancy_rate(worker_name, worker_group, window, rate as f64); + } + } + + if CONTAINER_SERIES_OWNER.get_or_init(|| worker_name.to_string()) != worker_name { + return; + } + + let memory_limit = *MEMORY_LIMIT.lock().unwrap().get_or_insert_with(get_memory); + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + let labels = &[worker_name, worker_group]; + if let Some(v) = memory_usage { + m.memory_usage.with_label_values(labels).set(v); + } + if let Some(v) = wm_memory_usage { + m.windmill_memory_usage.with_label_values(labels).set(v); + } + if let Some(v) = memory_limit { + m.memory_limit.with_label_values(labels).set(v); + } + } + otel_set_worker_memory( + worker_name, + worker_group, + memory_usage, + wm_memory_usage, + memory_limit, + ); + + let Some(stat) = get_cpu_stat() else { + return; + }; + let delta = { + let mut last = LAST_CPU_STAT.lock().unwrap(); + // A reading below the previous one means the cgroup changed under the process: + // restart from it rather than exporting a negative increase. + let delta = if stat.usage_usec < last.usage_usec { + CpuStat::default() + } else { + CpuStat { + usage_usec: stat.usage_usec - last.usage_usec, + nr_periods: stat.nr_periods.saturating_sub(last.nr_periods), + nr_throttled: stat.nr_throttled.saturating_sub(last.nr_throttled), + throttled_usec: stat.throttled_usec.saturating_sub(last.throttled_usec), + } + }; + *last = stat; + delta + }; + let usage_secs = delta.usage_usec as f64 / 1e6; + let throttled_secs = delta.throttled_usec as f64 / 1e6; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + let labels = &[worker_name, worker_group]; + m.cpu_usage.with_label_values(labels).inc_by(usage_secs); + m.cpu_periods + .with_label_values(labels) + .inc_by(delta.nr_periods); + m.cpu_throttled_periods + .with_label_values(labels) + .inc_by(delta.nr_throttled); + m.cpu_throttled_seconds + .with_label_values(labels) + .inc_by(throttled_secs); + } + otel_add_worker_cpu( + worker_name, + worker_group, + usage_secs, + delta.nr_periods, + delta.nr_throttled, + throttled_secs, + ); +} + +/// `mem_peak_kb` is the job's `mem_peak`: non-positive when the executor had no process +/// to sample or the reading failed, and those are not observations. +pub(crate) fn record_job_memory_peak(tag: &str, mem_peak_kb: i32) { + if mem_peak_kb <= 0 { + return; + } + let bytes = mem_peak_kb as u64 * 1024; + #[cfg(feature = "prometheus")] + if let Some(m) = PROM_METRICS.as_ref() { + m.job_memory_peak + .with_label_values(&[tag]) + .observe(bytes as f64); + } + otel_record_job_memory_peak(tag, bytes); +} diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 0490ed1754..e5b84f39b9 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -21,6 +21,7 @@ use windmill_common::{ error::{self, Error}, flow_status::FlowJobDuration, jobs::JobKind, + runnable_job_stats::{accumulate_runnable_job_stats, flush_runnable_job_stats}, utils::WarnAfterExt, worker::{error_to_value, to_raw_value, Connection, WORKER_GROUP}, worker_group_job_stats::{accumulate_job_stats, flush_stats_to_db, JobStatsMap}, @@ -32,8 +33,8 @@ use windmill_common::bench::{BenchmarkInfo, BenchmarkIter}; use windmill_queue::{ append_logs, asset_dispatch, get_mini_completed_job, is_pre_shaped_wm_failure_result, - parse_result_object, CanceledBy, FlowRunners, JobCompleted, MiniCompletedJob, MiniPulledJob, - ValidableJson, WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, + parse_result_object, CanceledBy, FlowRunners, JobCompleted, JobResourceUsage, MiniCompletedJob, + MiniPulledJob, ValidableJson, WrappedError, INIT_SCRIPT_TAG, MANUAL_FAILURE_ERROR_NAME, }; use serde_json::{json, value::RawValue, Value}; @@ -231,6 +232,10 @@ async fn process_jc( let duration_ms = jc.duration.clone(); let script_lang = jc.job.script_lang.clone(); let workspace_id = jc.job.workspace_id.clone(); + let job_kind = jc.job.kind; + let runnable_path = jc.job.runnable_path.clone(); + let mem_peak = jc.mem_peak; + let resource_usage = jc.resource_usage.take(); let root_job = handle_receive_completed_job( jc, @@ -269,6 +274,17 @@ async fn process_jc( duration_ms, ) .await; + accumulate_runnable_job_stats( + &workspace_id, + job_kind, + runnable_path.as_deref(), + resource_usage + .as_ref() + .map_or(WORKER_GROUP.as_str(), |u| u.worker_group.as_str()), + duration_ms, + mem_peak, + resource_usage.as_ref().and_then(|u| u.cpu_time_ms), + ); } success @@ -317,6 +333,14 @@ pub fn start_background_processor( let flush_handle = tokio::spawn(async move { let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(900)); // Flush every 15 min + // Once per hourly bucket, at an offset of its own per worker so that a fleet + // started together does not write the same rows in the same second. + let runnable_period = tokio::time::Duration::from_secs(3600); + let mut runnable_interval = tokio::time::interval_at( + tokio::time::Instant::now() + runnable_period.mul_f64(rand::random::()), + runnable_period, + ); + loop { tokio::select! { _ = interval.tick() => { @@ -324,6 +348,11 @@ pub fn start_background_processor( tracing::error!("Failed to flush worker group job stats: {}", e); } } + _ = runnable_interval.tick() => { + if let Err(e) = flush_runnable_job_stats(&db_clone).await { + tracing::error!("Failed to flush runnable job stats: {}", e); + } + } _ = killpill_rx_clone.recv() => { tracing::info!("bg processor received killpill signal, flushing remaining stats"); break; @@ -524,6 +553,18 @@ pub fn start_background_processor( Ok(Err(join_err)) => tracing::error!("Stats flush task failed: {}", join_err), Err(_) => tracing::error!("Stats flush timed out after 10 seconds"), } + // The periodic task stops at the killpill, before the jobs drained above were + // accumulated: without this a short-lived worker would lose its whole rollup. + match tokio::time::timeout( + std::time::Duration::from_secs(10), + flush_runnable_job_stats(&db), + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(e)) => tracing::error!("Failed to flush runnable job stats: {}", e), + Err(_) => tracing::error!("Runnable job stats flush timed out after 10 seconds"), + } job_completed_processor_is_done.store(true, Ordering::SeqCst); @@ -555,6 +596,7 @@ pub async fn process_result( job_dir: &str, job_completed_tx: JobCompletedSender, mem_peak: i32, + cpu_time_ms: Option, canceled_by: Option, cached_res_path: Option, token: &str, @@ -565,6 +607,7 @@ pub async fn process_result( has_stream: bool, flow_runners: Option>, ) -> error::Result { + let resource_usage = Some(JobResourceUsage { worker_group: WORKER_GROUP.clone(), cpu_time_ms }); match result { Ok(result) => { send_job_completed( @@ -575,6 +618,7 @@ pub async fn process_result( result, result_columns, mem_peak, + resource_usage, canceled_by, success: true, cached_res_path, @@ -658,6 +702,7 @@ pub async fn process_result( result_columns: None, preprocessed_args: None, mem_peak, + resource_usage, canceled_by, success: false, cached_res_path, @@ -1776,7 +1821,7 @@ pub async fn process_completed_job( add_time!(bench, "pre add_completed_job"); - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &job, true, @@ -1835,7 +1880,10 @@ pub async fn process_completed_job( true, canceled_by, result, - started_at.map(|x| FlowJobDuration { started_at: x, duration_ms: duration }), + span.started_at.or(started_at).map(|x| FlowJobDuration { + started_at: x, + duration_ms: span.duration_ms, + }), StepFailureKind::Normal, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, @@ -1867,9 +1915,10 @@ pub async fn process_completed_job( // `wm_failure` field) so a real runtime failure whose raw // result happens to contain a `wm_failure` field still goes // through the standard `WrappedError { error: ... }` wrap path. - let downstream_result: Arc> = if is_pre_shaped_wm_failure_result(result.get()) - { - windmill_queue::add_completed_job_pre_shaped_failure( + let (downstream_result, span): (Arc>, _) = if is_pre_shaped_wm_failure_result( + result.get(), + ) { + let span = windmill_queue::add_completed_job_pre_shaped_failure( db, &job, mem_peak.to_owned(), @@ -1880,9 +1929,9 @@ pub async fn process_completed_job( None, ) .await?; - result.clone() + (result.clone(), span) } else { - let wrapped = add_completed_job_error( + let (wrapped, span) = windmill_queue::add_completed_job_error_with_span( db, &job, mem_peak.to_owned(), @@ -1895,7 +1944,10 @@ pub async fn process_completed_job( None, ) .await?; - Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap()) + ( + Arc::new(serde_json::value::to_raw_value(&wrapped).unwrap()), + span, + ) }; #[cfg(all(feature = "enterprise", feature = "private"))] if job.kind == JobKind::DeploymentCallback { @@ -1927,12 +1979,19 @@ pub async fn process_completed_job( false, canceled_by, downstream_result, - duration.and_then(|d| { - job.started_at.map(|started_at| FlowJobDuration { - started_at: started_at, - duration_ms: d, + if span.outlasts_run(job.started_at) { + span.started_at.map(|started_at| FlowJobDuration { + started_at, + duration_ms: span.duration_ms, }) - }), + } else { + duration.and_then(|d| { + job.started_at.map(|started_at| FlowJobDuration { + started_at: started_at, + duration_ms: d, + }) + }) + }, StepFailureKind::Normal, &same_worker_tx.expect(SAME_WORKER_REQUIREMENTS).to_owned(), &worker_dir, diff --git a/backend/windmill-worker/src/wac_executor.rs b/backend/windmill-worker/src/wac_executor.rs index 45d76c74e8..3cc9c1e23a 100644 --- a/backend/windmill-worker/src/wac_executor.rs +++ b/backend/windmill-worker/src/wac_executor.rs @@ -113,6 +113,11 @@ pub enum WacPark { /// handler — falls back to `now() - started_at`. Left pointing at the first segment, /// that fallback reports the whole sleep or approval wait as execution time. /// +/// The first park keeps that first start in `extras.wac_first_started_at`, so the +/// completed row can still say when the workflow began and how long it took overall +/// (`Completion::execute`). The queue row's `started_at` stays per segment. The inline +/// checkpoint requeue clears `started_at` on its own and keeps the first start the same way. +/// /// Call it before any write to the parent's `v2_job_status` row in the same /// transaction: a child's completion locks the queue row and then the status row /// (`record_child_completion`), and taking them the other way round here can @@ -156,7 +161,10 @@ pub async fn suspend_wac_parent( sqlx::query!( "UPDATE v2_job_queue - SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null + SET suspend = $3, suspend_until = now() + make_interval(secs => $4), started_at = null, + extras = CASE WHEN started_at IS NULL OR extras ? 'wac_first_started_at' THEN extras + ELSE jsonb_set(COALESCE(extras, '{}'::jsonb), '{wac_first_started_at}', to_jsonb(started_at)) + END WHERE id = $1 AND workspace_id = $2", job_id, w_id, diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index d0d5d8b4e2..7c3c3971fa 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -215,6 +215,8 @@ lazy_static::lazy_static! { pub static ref PY311_CACHE_DIR: String = format!("{}python_3_11", *ROOT_CACHE_DIR); pub static ref PY312_CACHE_DIR: String = format!("{}python_3_12", *ROOT_CACHE_DIR); pub static ref PY313_CACHE_DIR: String = format!("{}python_3_13", *ROOT_CACHE_DIR); + pub static ref PY314_CACHE_DIR: String = format!("{}python_3_14", *ROOT_CACHE_DIR); + pub static ref PY315_CACHE_DIR: String = format!("{}python_3_15", *ROOT_CACHE_DIR); pub static ref TAR_JAVA_CACHE_DIR: String = format!("{}tar/java", *ROOT_CACHE_DIR); @@ -2247,6 +2249,7 @@ pub async fn handle_all_job_kind_error( ))), result_columns: None, mem_peak: 0, + resource_usage: None, canceled_by: None, success: false, cached_res_path: None, @@ -3772,6 +3775,7 @@ pub async fn run_worker( result: Arc::new(empty_result()), result_columns: None, mem_peak: 0, + resource_usage: None, cached_res_path: None, token: "".to_string(), canceled_by: None, @@ -4763,6 +4767,7 @@ pub async fn handle_queued_job( result, result_columns: None, mem_peak: 0, + resource_usage: None, canceled_by: None, success: true, cached_res_path: None, @@ -5112,6 +5117,8 @@ pub async fn handle_queued_job( let cjob = MiniCompletedJob::from(job.to_owned()); drop(job); + // Taken ahead of the returns below so that none of them leaves the entry behind. + let cpu_time_ms = crate::handle_child::take_job_cpu_time_ms(&cjob.id); //it's a test job, no need to update the db if cjob.workspace_id == "" { return Ok(JobOutcome::Completed); @@ -5128,14 +5135,27 @@ pub async fn handle_queued_job( .is_err_and(|err| matches!(err, &Error::WacSuspended(_))) { // WAC v2 job suspended while waiting for child jobs — don't complete it + // Only a worker with a database connection flushes the rollup. + if let Connection::Sql(_) = conn { + windmill_common::runnable_job_stats::accumulate_runnable_round( + &cjob.workspace_id, + cjob.kind, + cjob.runnable_path.as_deref(), + &WORKER_GROUP, + started.elapsed().as_millis() as i64, + cpu_time_ms, + ); + } return Ok(JobOutcome::Completed); } + crate::resource_metrics::record_job_memory_peak(&cjob.tag, mem_peak); process_result( cjob, result.map(|x| Arc::new(x)), job_dir, job_completed_tx, mem_peak, + cpu_time_ms, canceled_by, cached_res_path, &client.token, diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 6d76d0d771..4f67780f18 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -2097,7 +2097,7 @@ pub async fn update_flow_status_after_job_completion_internal( ) .await?; let duration = if success { - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &cflow_job, true, @@ -2111,9 +2111,9 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + span.duration_ms } else { - let (_, duration) = add_completed_job( + let (_, span) = add_completed_job( db, &cflow_job, false, @@ -2131,7 +2131,7 @@ pub async fn update_flow_status_after_job_completion_internal( false, ) .await?; - duration + span.duration_ms }; flow_job_duration = flow_job .started_at diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index b608601991..6189d39b8d 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -1244,6 +1244,7 @@ pub async fn handle_flow_dependency_job( &job.workspace_id, flow.failure_module.as_ref(), flow.same_worker, + flow.preserve_step_tags, ) .await?; @@ -2134,6 +2135,7 @@ async fn insert_flow_modules<'c>( workspace_id: &str, failure_module: Option<&Box>, same_worker: bool, + preserve_step_tags: bool, modules: &mut Vec, modules_node: &mut Option, ) -> Result> { @@ -2144,6 +2146,7 @@ async fn insert_flow_modules<'c>( workspace_id, failure_module, same_worker, + preserve_step_tags, )) .await?; if modules.is_empty() || crate::worker_flow::is_simple_modules(modules, failure_module) { @@ -2160,6 +2163,7 @@ async fn insert_flow_modules<'c>( modules: std::mem::take(modules), failure_module: failure_module.cloned(), same_worker, + preserve_step_tags, ..Default::default() }))), None, @@ -2176,6 +2180,7 @@ async fn reduce_flow<'c>( workspace_id: &str, failure_module: Option<&Box>, same_worker: bool, + preserve_step_tags: bool, ) -> Result> { use FlowModuleValue::*; for module in &mut *modules { @@ -2234,6 +2239,7 @@ async fn reduce_flow<'c>( workspace_id, failure_module, same_worker, + preserve_step_tags, modules, modules_node, ) @@ -2247,6 +2253,7 @@ async fn reduce_flow<'c>( workspace_id, failure_module, same_worker, + preserve_step_tags, &mut branch.modules, &mut branch.modules_node, ) @@ -2258,6 +2265,7 @@ async fn reduce_flow<'c>( workspace_id, failure_module, same_worker, + preserve_step_tags, default, default_node, ) @@ -2271,6 +2279,7 @@ async fn reduce_flow<'c>( workspace_id, failure_module, same_worker, + preserve_step_tags, &mut branch.modules, &mut branch.modules_node, ) diff --git a/backend/windmill-worker/src/worker_utils.rs b/backend/windmill-worker/src/worker_utils.rs index cabf004f4d..c634c13b6a 100644 --- a/backend/windmill-worker/src/worker_utils.rs +++ b/backend/windmill-worker/src/worker_utils.rs @@ -39,17 +39,26 @@ pub(crate) async fn update_worker_ping_full( let wm_memory_usage = get_windmill_memory_usage(); let (vcpus, memory) = if read_cgroups { - (get_vcpus(), get_memory()) + let memory = get_memory(); + crate::resource_metrics::set_memory_limit(memory); + (get_vcpus(), memory) } else { (None, None) }; + let occupancy = occupancy_metrics.update_occupancy_metrics(); + crate::resource_metrics::record_worker_resources( + worker_name, + memory_usage, + wm_memory_usage, + Some(&occupancy), + ); let OccupancyResult { occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, - } = occupancy_metrics.update_occupancy_metrics(); + } = occupancy; let ping_start = std::time::Instant::now(); if let Err(e) = (|| { diff --git a/benchmarks/README.md b/benchmarks/README.md index fc758c006f..1dc4818848 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -22,12 +22,12 @@ deno run -A benchmark_suite.ts -c suite_wac.json ### Script benchmarks - `noop` — Empty jobs (measures pure scheduling overhead) -- `deno`, `bun`, `python`, `go`, `bash` — Language runtimes +- `bun`, `python`, `go`, `bash` — Language runtimes - `nativets` — BunNative (no isolation) - `dedicated`, `dedicated_nativets` — Dedicated worker mode ### Flow benchmarks -- `2steps` — 2-step flow (deno + identity) +- `2steps` — 2-step flow (bun + identity) - `bigscriptinflow` — Flow with large raw bash script - `flow_seq_2_bun` — 2 sequential bun steps - `flow_par_2_bun` — 2 parallel bun steps (branchall) diff --git a/benchmarks/benchmark_oneoff.ts b/benchmarks/benchmark_oneoff.ts index 5f075d2034..5aa75b5864 100644 --- a/benchmarks/benchmark_oneoff.ts +++ b/benchmarks/benchmark_oneoff.ts @@ -37,7 +37,7 @@ async function verifyOutputs(uuids: string[], workspace: string) { console.log(`Incorrect results: ${incorrectResults}`); } -export const NON_TEST_TAGS = ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets", "flow"] +export const NON_TEST_TAGS = ["python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets", "flow"] const FLOW_COMPARISON_KINDS = ["flow_seq_2_bun", "flow_par_2_bun", "flow_seq_3_bun"]; export async function main({ @@ -148,7 +148,7 @@ export async function main({ } if ( - ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes( + ["python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes( kind ) ) { @@ -169,7 +169,7 @@ export async function main({ kind: "noop", }); } else if ( - ["deno", "python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes( + ["python", "go", "bash", "dedicated", "bun", "nativets", "dedicated_nativets"].includes( kind ) ) { @@ -414,7 +414,7 @@ if (import.meta.main) { ) .option( "--kind ", - "Specifiy the benchmark kind among: deno, identity, python, go, bash, dedicated, bun, noop, 2steps, nativets, dedicated_nativets, wac_seq_2, wac_par_2, wac_seq_3, wac_inline_2, flow_seq_2_bun, flow_par_2_bun, flow_seq_3_bun", + "Specifiy the benchmark kind among: identity, python, go, bash, dedicated, bun, noop, 2steps, nativets, dedicated_nativets, wac_seq_2, wac_par_2, wac_seq_3, wac_inline_2, flow_seq_2_bun, flow_par_2_bun, flow_seq_3_bun", { required: true, } diff --git a/benchmarks/graphs_config.json b/benchmarks/graphs_config.json index 8e37695d66..61b5bdddc8 100644 --- a/benchmarks/graphs_config.json +++ b/benchmarks/graphs_config.json @@ -49,26 +49,6 @@ } ] }, - { - "graph_title": "deno throughput benchmark", - "benchmarks": [ - { - "kind": "deno", - "workers": 1, - "label": "1 worker" - }, - { - "kind": "deno", - "workers": 4, - "label": "4 workers" - }, - { - "kind": "deno", - "workers": 8, - "label": "8 workers" - } - ] - }, { "graph_title": "bun throughput benchmark", "benchmarks": [ @@ -170,7 +150,7 @@ ] }, { - "graph_title": "go vs python vs deno vs bun vs bash vs nativets", + "graph_title": "go vs python vs bun vs bash vs nativets", "benchmarks": [ { "kind": "go", @@ -182,11 +162,6 @@ "workers": 1, "label": "python" }, - { - "kind": "deno", - "workers": 1, - "label": "deno" - }, { "kind": "bun", "workers": 1, diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index df2044d419..34e4da5cf3 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.828.0"; +export const VERSION = "v1.830.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ @@ -87,10 +87,6 @@ export async function createBenchScript( schemaProperties = { uuid: { default: null, description: "", type: "string" }, }; - } else if (scriptPattern === "deno") { - scriptContent = - 'export function main(){ return Deno.env.get("WM_JOB_ID"); }'; - language = "deno"; } else if (scriptPattern === "nativets") { scriptContent = '//native\nexport async function main(){ return (await fetch(BASE_URL + "/api/version")).text() }'; @@ -256,10 +252,10 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { id: "a", value: { input_transforms: {}, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: - 'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }', + 'export function main(){ return Bun.env["WM_FLOW_JOB_ID"]; }', }, }, { @@ -277,7 +273,7 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { expr: "results.a", }, }, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: "export function main(x: string){ return x; }", }, @@ -298,10 +294,10 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { id: "a", value: { input_transforms: {}, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: - 'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }', + 'export function main(){ return Bun.env["WM_FLOW_JOB_ID"]; }', }, }, { @@ -321,7 +317,7 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { expr: "results.a", }, }, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: "export function main(x: string){ return x; }", }, @@ -339,7 +335,7 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { expr: "results.a", }, }, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: "export function main(x: string){ return x; }", }, @@ -490,10 +486,10 @@ export const getFlowPayload = (flowPattern: string): api.FlowPreview => { id: "a", value: { input_transforms: {}, - language: api.RawScript.language.DENO, + language: "bun" as api.RawScript.language, type: "rawscript", content: - 'export function main(){ return Deno.env.get("WM_FLOW_JOB_ID"); }', + 'export function main(){ return Bun.env["WM_FLOW_JOB_ID"]; }', }, }, { diff --git a/benchmarks/main.ts b/benchmarks/main.ts index 1f681a9c03..bf2f998854 100644 --- a/benchmarks/main.ts +++ b/benchmarks/main.ts @@ -175,11 +175,11 @@ export async function main({ if ( !useFlows && (scriptPattern === undefined || - ["deno", "python", "go", "bash", "bun", "dedicated"].includes( + ["python", "go", "bash", "bun", "dedicated"].includes( scriptPattern )) ) { - await createBenchScript(scriptPattern || "deno", workspace); + await createBenchScript(scriptPattern || "bun", workspace); } let workers: Worker[] = new Array(num_workers); @@ -458,7 +458,7 @@ if (import.meta.main) { ) .option( "--script-pattern ", - "Use a different script pattern among: deno, identity, python, go, bash, dedicated, bun (Default deno)" + "Use a different script pattern among: bun, identity, python, go, bash, dedicated (Default bun)" ) .option("--custom ", "Use custom actions during bench") .option( diff --git a/benchmarks/suite_config.json b/benchmarks/suite_config.json index a2704a46fe..ed50c60b1e 100644 --- a/benchmarks/suite_config.json +++ b/benchmarks/suite_config.json @@ -7,10 +7,6 @@ "kind": "2steps", "jobs": 250 }, - { - "kind": "deno", - "jobs": 500 - }, { "kind": "bun", "jobs": 500 diff --git a/benchmarks/worker.ts b/benchmarks/worker.ts index ae06fa9779..ce2d695771 100644 --- a/benchmarks/worker.ts +++ b/benchmarks/worker.ts @@ -118,7 +118,7 @@ while (cont) { } else { uuid = await windmill.JobService.runScriptByPath({ workspace: config.workspace_id, - path: "f/benchmarks/" + (config.scriptPattern || "deno"), + path: "f/benchmarks/" + (config.scriptPattern || "bun"), requestBody: {}, }); } diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json index c6691e0048..03a55cff86 100644 --- a/chat-sdk/package-lock.json +++ b/chat-sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-chat", - "version": "1.828.0", + "version": "1.830.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-chat", - "version": "1.828.0", + "version": "1.830.0", "license": "Apache-2.0", "devDependencies": { "@ai-sdk/react": "^4.0.102", diff --git a/chat-sdk/package.json b/chat-sdk/package.json index a119b8645d..96637c376c 100644 --- a/chat-sdk/package.json +++ b/chat-sdk/package.json @@ -1,7 +1,7 @@ { "name": "windmill-chat", "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", - "version": "1.828.0", + "version": "1.830.0", "author": "Ruben Fiszel", "license": "Apache-2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", diff --git a/cli/src/commands/init/template.ts b/cli/src/commands/init/template.ts index 8a433d69d4..540d8c7978 100644 --- a/cli/src/commands/init/template.ts +++ b/cli/src/commands/init/template.ts @@ -77,8 +77,8 @@ const WORKSPACE_CONFIG_SCHEMA = { */ export const CONFIG_REFERENCE: ConfigOption[] = [ // ── Core ────────────────────────────────────────────────────────────── - { name: "defaultTs", type: "string", enum: ["bun", "deno"], default: "bun", description: "Default TypeScript runtime for new scripts", - inlineComment: "bun or deno" }, + { name: "defaultTs", type: "string", enum: ["bun", "deno"], default: "bun", description: "TypeScript runtime of a plain .ts script (bun when absent)", + skipInTemplate: true }, { name: "includes", type: "array", items: { type: "string" }, default: '["f/**"]', description: "Glob patterns for files to include in sync", templateValue: '\n - "f/**"', inlineComment: 'use ** for all, f/** for folder-scoped' }, { name: "extraIncludes", type: "array", items: { type: "string" }, default: "[]", description: "Additional glob patterns merged with includes (useful in workspace overrides)", diff --git a/cli/src/core/conf.ts b/cli/src/core/conf.ts index 455e876292..1ad0b69c1b 100644 --- a/cli/src/core/conf.ts +++ b/cli/src/core/conf.ts @@ -320,12 +320,6 @@ export async function readConfigFile(opts?: { warnIfMissing?: boolean }): Promis delete (conf as any)?.environments; delete conf?.git_branches; - if (conf?.defaultTs == undefined) { - log.warn( - "No defaultTs defined in your wmill.yaml. Using 'bun' as default." - ); - } - // Initialize global nonDottedPaths setting from config setNonDottedPaths(conf?.nonDottedPaths ?? false); @@ -376,7 +370,6 @@ export const DEFAULT_SYNC_OPTIONS: Readonly< Required< Pick< SyncOptions, - | "defaultTs" | "includes" | "excludes" | "codebases" @@ -401,7 +394,6 @@ export const DEFAULT_SYNC_OPTIONS: Readonly< > > > = { - defaultTs: "bun", includes: ["f/**"], excludes: [], codebases: [], diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 860a3ddede..21aeb07f60 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.828.0"; +export const VERSION = "1.830.0"; diff --git a/cli/src/core/settings.ts b/cli/src/core/settings.ts index 57e89f5ded..75dfa3b8da 100644 --- a/cli/src/core/settings.ts +++ b/cli/src/core/settings.ts @@ -413,7 +413,7 @@ export async function pushWorkspaceSettings( const localOperatorSettings = localSettings.operator_settings && { ...localSettings.operator_settings, }; - for (const key of ["builder_flows", "manage_schedules", "manage_triggers"] as const) { + for (const key of ["builder_flows", "builder_apps", "manage_schedules", "manage_triggers"] as const) { const remote = settings.operator_settings?.[key]; if (localOperatorSettings && localOperatorSettings[key] === undefined && remote !== undefined) { localOperatorSettings[key] = remote; diff --git a/cli/src/utils/script_common.ts b/cli/src/utils/script_common.ts index 611268c27c..e3deabb6a7 100644 --- a/cli/src/utils/script_common.ts +++ b/cli/src/utils/script_common.ts @@ -189,6 +189,8 @@ const LANG_ANNOTATIONS: Partial> = { "py311", "py312", "py313", + "py314", + "py315", "sandbox", ], bun: ["npm", "nodejs", "native", "nobundling", "sandbox"], @@ -208,7 +210,7 @@ const LANG_ANNOTATIONS: Partial> = { * * `# py: ` is the exception the macro does not cover: the python * import parser reads it directly (`windmill-parser-py-imports`, alongside the - * `py310`..`py313` flags) to pick the interpreter, which changes what resolves. + * `py310`..`py315` flags) to pick the interpreter, which changes what resolves. */ export function hasLockAffectingAnnotation( scriptContent: string, diff --git a/cli/test/init_no_git_sync.test.ts b/cli/test/init_no_git_sync.test.ts index d16c9e771f..189e82562f 100644 --- a/cli/test/init_no_git_sync.test.ts +++ b/cli/test/init_no_git_sync.test.ts @@ -56,8 +56,6 @@ test("Init: createWorkspaceProfile includes defaults when no repositories exist" expect(Array.isArray(workspaceProfile.excludes)).toBeTruthy(); expect(workspaceProfile.excludes.length).toEqual(0); - expect(workspaceProfile.defaultTs).toEqual('bun'); - console.log('Workspace profile correctly includes default sync settings when no repositories exist'); }); @@ -74,8 +72,6 @@ test("Init: verify DEFAULT_SYNC_OPTIONS has expected values", () => { expect(Array.isArray(DEFAULT_SYNC_OPTIONS.excludes)).toBeTruthy(); expect(DEFAULT_SYNC_OPTIONS.excludes.length).toEqual(0); - expect(DEFAULT_SYNC_OPTIONS.defaultTs).toEqual('bun'); - console.log('DEFAULT_SYNC_OPTIONS has expected values'); }); @@ -178,7 +174,7 @@ test.skipIf(shouldSkipOnCI())("Init: --use-default bypasses backend settings che // Should have default settings, not backend settings expect(wmillYaml).toContain("includes:\n - f/**"); - expect(wmillYaml).toContain("defaultTs: bun"); + expect(wmillYaml).not.toContain("defaultTs"); // Should NOT have backend-specific settings expect(wmillYaml.includes("f/should-be-ignored/**")).toEqual(false); diff --git a/cli/test/init_template_unit.test.ts b/cli/test/init_template_unit.test.ts index 227fc271a2..7070de63c8 100644 --- a/cli/test/init_template_unit.test.ts +++ b/cli/test/init_template_unit.test.ts @@ -90,7 +90,6 @@ describe("generateCommentedTemplate", () => { test("default values match expected defaults", () => { const config = parse(generateCommentedTemplate("main")); - expect(config.defaultTs).toBe("bun"); expect(config.skipSecrets).toBe(true); expect(config.nonDottedPaths).toBe(true); expect(config.codebases).toEqual([]); diff --git a/cli/wmill.schema.json b/cli/wmill.schema.json index 022156e5c9..145a75fd26 100644 --- a/cli/wmill.schema.json +++ b/cli/wmill.schema.json @@ -10,7 +10,7 @@ "bun", "deno" ], - "description": "Default TypeScript runtime for new scripts" + "description": "TypeScript runtime of a plain .ts script (bun when absent)" }, "includes": { "type": "array", diff --git a/docker/DockerfileFull b/docker/DockerfileFull index 8ef6be78bb..c3a82c8312 100644 --- a/docker/DockerfileFull +++ b/docker/DockerfileFull @@ -1,6 +1,9 @@ ARG BASE_IMAGE=ghcr.io/windmill-labs/windmill:dev FROM ${BASE_IMAGE} +# Deno +COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno + # Rust COPY --from=rust:1.97.0 /usr/local/cargo /usr/local/cargo COPY --from=rust:1.97.0 /usr/local/rustup /usr/local/rustup @@ -12,7 +15,9 @@ RUN RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo /usr/local/cargo/b # ansible there leaves its venv interpreter as a dangling symlink at runtime, so every # ansible-* executable fails with ENOENT ("ansible-galaxy not found"). Pin the tool's # interpreter to a persistent image path so the install stays self-contained. -RUN UV_PYTHON_INSTALL_DIR=/usr/local/uv/py uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -sf -t "$UV_TOOL_BIN_DIR/" || true +# The version is explicit because uv otherwise downloads its newest stable Python, which +# ansible-core may not support yet: the install succeeds and every executable fails on import. +RUN UV_PYTHON_INSTALL_DIR=/usr/local/uv/py uv tool install --python 3.14 ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -sf -t "$UV_TOOL_BIN_DIR/" || true # C# RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \ diff --git a/docker/DockerfileFullEe b/docker/DockerfileFullEe index 88ed619c19..bb83195f62 100644 --- a/docker/DockerfileFullEe +++ b/docker/DockerfileFullEe @@ -21,6 +21,9 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \ FROM ${BASE_IMAGE} +# Deno +COPY --from=denoland/deno:2.2.1 --chmod=755 /usr/bin/deno /usr/bin/deno + # Rust COPY --from=rust:1.97.0 /usr/local/cargo /usr/local/cargo COPY --from=rust:1.97.0 /usr/local/rustup /usr/local/rustup @@ -32,7 +35,9 @@ RUN RUSTUP_HOME=/usr/local/rustup CARGO_HOME=/usr/local/cargo /usr/local/cargo/b # ansible there leaves its venv interpreter as a dangling symlink at runtime, so every # ansible-* executable fails with ENOENT ("ansible-galaxy not found"). Pin the tool's # interpreter to a persistent image path so the install stays self-contained. -RUN UV_PYTHON_INSTALL_DIR=/usr/local/uv/py uv tool install ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -sf -t "$UV_TOOL_BIN_DIR/" || true +# The version is explicit because uv otherwise downloads its newest stable Python, which +# ansible-core may not support yet: the install succeeds and every executable fails on import. +RUN UV_PYTHON_INSTALL_DIR=/usr/local/uv/py uv tool install --python 3.14 ansible && [ -d "$(uv tool dir)/ansible/bin/" ] && find "$(uv tool dir)/ansible/bin/" -mindepth 1 -maxdepth 1 -type f -executable -regextype posix-extended -regex '^((.+/)?)[^.]+' -print0 | xargs -0 ln -sf -t "$UV_TOOL_BIN_DIR/" || true # dotnet SDK RUN wget https://dot.net/v1/dotnet-install.sh -O dotnet-install.sh \ && chmod +x dotnet-install.sh \ diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index 45986bbd5f..a37ced0eca 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -58,7 +58,7 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmo ENV TZ=Etc/UTC # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtime to temp location (will copy with world-writable perms later) # --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 5ac24f5c1d..687b1506ce 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -58,7 +58,7 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmo ENV TZ=Etc/UTC # Install UV -RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.11.24/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.13.0/uv-installer.sh | sh && mv /root/.local/bin/uv /usr/local/bin/uv # Preinstall python runtime to temp location (will copy with world-writable perms later) # --compile-bytecode precompiles the stdlib to .pyc so jobs don't recompile it on every run diff --git a/docs/operator-builder-rights.md b/docs/operator-builder-rights.md index 4db2a69511..4f844beebf 100644 --- a/docs/operator-builder-rights.md +++ b/docs/operator-builder-rights.md @@ -1,13 +1,25 @@ # Operator builder rights -`operator_settings.builder_flows` lets every operator of a workspace compose flows out of runnables -that already exist. It does not make them authors: the boundary the operator role draws is -**authoring code and running arbitrary code**, and this does not move it. +Two workspace settings, `operator_settings.builder_flows` and `operator_settings.builder_apps`, +that let every operator of that workspace compose flows and full-code apps out of runnables that +already exist. Neither lets them author backend code or run arbitrary code on a worker, the +boundary the operator role draws. The app right does let them write the app's frontend code, +which runs in the viewer's browser and is why that right rests on sandbox isolation. -It is a write right, unlike the visibility flags beside it, and unlike the withdrawable rights in -`docs/operator-write-rights.md` it is granted on request and costs a seat. Read it with -`windmill_common::workspaces::operator_can_build_flows` (60s cache, shared with the withdrawable -rights) and gate a write with `check_operator_can_build_flows`. +They are granted **independently**, because the two are not equally verifiable. A composed flow is +checked in full server-side (`check_flow_is_composition_only` refuses everything carrying code). +A full-code app is a browser-built bundle no server-side check can read, and rests instead on +forced `sandbox` isolation plus the viewer consent prompt. An admin may reasonably want the first +without the second, so **never gate on "either"**: a gate that does authorizes the kind its +workspace never granted, which is the whole point of the split. `BuilderKind` exists to make each +call site say which one it means. + +These are write rights, unlike the visibility flags beside them, and unlike the withdrawable +rights in `docs/operator-write-rights.md` they are granted on request and cost a seat. Read them +with `windmill_common::workspaces::operator_builder_rights` (60s cache, shared with the +withdrawable rights), which returns an `OperatorBuilderRights { flows, apps }`, and gate a write +with `check_operator_can_build`, naming the `BuilderKind`. `.any()` is only for surfaces that are +genuinely not per-kind: the seat a membership costs, and listing drafts. ## What the check has to cover @@ -49,25 +61,131 @@ Call it on every write **and** every preview: `run_preview_flow_job` and it the way to run what the write path refuses. A flow draft goes through `validate_operator_flow` too: a developer who loads a builder's draft in the editor runs its code as themselves. +The same reasoning applies to a builder-authored app, with one extra step. `execute_component` +resolves the runnable it runs on the root handle, so `validate_operator_composed_app` checks every +referenced path under the caller's RLS and refuses hub ones. For a script it checks the version a +run would pick: grants are per version, and a path reused after its versions were archived starts +with none while the archived ones keep theirs. But it has to check **two** surfaces, +because they are not the same list: the policy's `script/` and `flow/` triggerables, +and the by-path entries of the app value's `runnables` map, which is what the editor resolves a +`runnable_id` against and sends. Read only that map, and refuse an entry whose `type`, `path` or +`runType` is not text. + +Reading a triggerable key is not a `split_once(':')`: `execute_component` looks up +`format!("{component}:{path}")` with an unrestricted component string, so `a:b:script/x` resolves +at run time for `component = "a:b"`. Every colon is a possible split, so the check validates every +suffix that parses as a runnable rather than guessing which one a request will use. + +What makes those checks bind is that **`ExecutionMode::Viewer` is refused for a builder app**. In +Viewer mode `execute_component` falls back to a default triggerable for any `script/`/`flow/` +path, so the policy stops being the list of what the app may invoke, and the job runs as the +*viewer*: an admin who merely opened the app would run anything in the workspace as themselves. +`Publisher`, `Guest` and `Anonymous` have no such fallback; all three run as the publisher. If you ever relax the Viewer refusal, the +deploy-time path checks above stop being an authorization boundary. + +## The two raw-app deploy paths are not equivalent + +`create_app_raw` / `update_app_raw` are multipart: the browser already built the bundle, and +nothing server-side compiles anything. Builders use these. + +`create_app_raw_source` / `update_app_raw_source` push a bundler CLI over caller-supplied `files` +as a job **on a worker**, which is arbitrary code execution and is why they already require +`jobs:run` on top of `apps:write`. They stay closed to operators, builder rights or not. Do not +"tidy" the exception away: it costs builders nothing, because the browser and the CLI both bundle +locally and deploy through the multipart endpoints. + +A builder-authored app is forced to `policy.sandbox = true`. That is what makes it safe to let an +operator publish a bundle nobody reviewed: without it the bundle runs same-origin with each +viewer's Windmill session. The same check refuses a `rawscript/` key in either triggerables +map, since that key is the deployed app's authorization to run caller-supplied `raw_code` hashing +to it. It reads the value as stored, NUL escapes stripped: a stripped NUL can turn a key it +ignored into `inlineScript`. + +`update_app` takes a builder too, but settings only: the deploy panel saves a deployed full-code +app's access and frontend scopes through it with no value. It reaches the same check, so a builder +can no more switch the sandbox off there than on deploy. + +A builder's **draft** goes through the same `check_operator_composed_app` as its deploy, with +`BuilderAppWrite::Draft`: an admin may open it in the editor or deploy it as it stands, so any +rule added for deploys must hold for drafts too. Where the two differ, `write` decides: a deploy +fills in an omitted sandbox or mode, since it stores the policy it checked, while a draft skips +`triggerables_v2` and must not name another user's run identity. Every frontend writer of a +builder draft must conform the policy first with `conformBuilderAppPolicy`, since a policy loaded +from someone else's app breaks these rules: the editor and the AI chat do. The AI Sessions raw-app +editor, open to operators only behind a per-browser opt-in, does not, so its autosaves are refused. + +The raw-app editor previews an app isolated only when its policy has `sandbox`, and otherwise runs +the bundle same-origin with the session of whoever opens it. A builder's draft is therefore +held to the same rule as its deploy: `update_draft` refuses one whose `policy.sandbox` is not +`true`, so whatever an admin opens of an operator's, deployed or drafted, previews isolated. +Turning the sandbox off in their own editor stays that admin's explicit choice. It also refuses +a draft naming a runnable the builder cannot read, or a hub one, like the deploy: the editor +preview runs them as whoever opens it, and admits any path for an admin. That covers the +policy's legacy `triggerables` too, `rawscript/` keys included: every deploy of a draft +carries it through untouched, so an admin deploying the draft would publish it. Its +`triggerables_v2` is not checked: every deploy of a draft rebuilds it from the runnables, the +preview never reads it, and a copy gone stale since the last deploy would otherwise block +autosave. `Viewer` mode is refused in a draft as on deploy: the deploy panel shows it as +members-only, so an admin deploying the draft would publish it unawares. The builder's editor +switches it to `Publisher` on load, since that panel offers no way out of it. A draft naming a +run identity other than its builder is refused for the same reason: a builder's own deploy resets +it, but an admin's one-click draft deploy preserves it without showing it. The builder's editor +drops another user's identity on load, which an app that user deployed carries. A builder in +`wm_deployers` is the exception: like any member, their deploy may keep another identity, which +was judged acceptable for operators in that group (WIN-2536) and is the same for composed flows. + +Reading a triggerable key at every colon copies it once per colon, so the check refuses a key +over 512 bytes before splitting: a quadratic walk otherwise lets a sub-megabyte key allocate +gigabytes. A 255-character path plus a component name stays well under it. + ## Billing -An operator of a builder workspace consumes a full author seat: composing deployable artifacts -makes them an author, and there is no half-author. `consumes_operator_seat` is the seat-role -helper; the EE counting queries share `OPERATOR_SEAT_SQL` so the displayed, enforced and reported -numbers agree. The one exception is `get_user_usage` in `stats_ee.rs`: it is a compile-checked -`query!`, which cannot interpolate the constant, so it spells the predicate out. Change both -together. +An operator of a builder workspace consumes a full author seat. This is the one place that reads +"either right", and deliberately so: either one makes an operator an author of deployable +artifacts, and there is no half-author, so granting both costs no more than granting one. Do not +introduce per-right pricing — it turns one boolean into a capability matrix that billing, the cap +check and the displayed count would each have to agree on. -On cloud, `billable_seats` applies the same rule; see its doc for the out-of-repo invoice it must -match. +`consumes_operator_seat` is the seat-role helper; the EE counting queries share +`OPERATOR_SEAT_SQL` so the displayed, enforced and reported numbers agree. The one exception is +`get_user_usage` in `stats_ee.rs`: it is a compile-checked `query!`, which cannot interpolate the +constant, so it spells the predicate out. Change both together. On cloud, `billable_seats` applies +the same rule; see its doc for the out-of-repo invoice it must match. -Granting the right runs `check_seat_cap_for_operator_builder`, which prices the change by counting -seats twice rather than by counting the workspace's operators: an operator who already authors -elsewhere must not be charged again, so re-saving settings that already have the right on is a -zero delta and never blocks. +Granting the first right runs `check_seat_cap_for_operator_builder`, which prices the change by +counting seats twice rather than by counting the workspace's operators: an operator who already +authors elsewhere must not be charged again. Adding the second right later, or re-saving settings +that already have one on, is a zero delta and never blocks. ## Accepted risks +- A builder raw app may declare `frontend_sdk_scopes`, and `mint_raw_app_sdk_token` mints as the + *viewer*. A consenting admin therefore hands the bundle a 12h admin-identity token within the + curated scope list. The viewer consent prompt is the gate, shown by the editor preview as well + as the deployed app, before any token is minted. The lever, if this is ever revisited, + is dropping `variables:read` / `resources:read` / `jobs:run` from `FRONTEND_SDK_ALLOWED_SCOPES` + for builder apps. +- The raw-app editor preview runs an app's runnables as whoever has the editor open, where the + deployed app runs them as its publisher. An admin opening a builder's app in the editor lets its + frontend code run, with the admin's rights, any runnable the builder can read. Running as the + draft's author instead would need a preview that ignores unsaved edits to the runnable list. +- A builder may make an app public (`ExecutionMode::Anonymous`): anyone with the URL runs the + runnables its policy names, as the builder. Refusing it is the workspace's call, through the + `RestrictAnonymousAppDeployment` protection rule. A `Guest` app is the same with a login in + front: anyone the instance's identity provider admits, member or not, once guest access is + enabled. `RestrictGuestAppDeployment` refuses it. +- A builder raw app may set `allow_user_resources` on a runnable's inputs, and the bundle picks the + `$res:` path it sends. `build_args` resolves that path as the *viewer*, with no prompt, into a + job that runs as the builder. Getting a value out takes a guessable path and a readable script + that returns, logs or sends its input. Developers can do the same; the lever is refusing a + non-empty `allow_user_resources` (and `s3_inputs[].allow_user_resources`) for builders. +- A builder's delete or settings-only update reads the app's kind before the write, not under a + lock held through it. An admin converting that app to low-code in between lets the builder's + request land on the low-code app; the builder already has write access to its path. +- A builder's draft carries no run identity other than the builder's own, and deploying one that + carries none makes the deployer its run identity, as any deploy does. An admin deploying such a + draft, from the editor or Compare & Deploy, therefore publishes the builder's code running as + that admin; the deploy is the admin's explicit act, as it is for a developer's draft. - All-or-nothing per workspace: there is no per-user builder role. - `operator_settings` is git-synced, so a pull can flip every operator's class in a workspace and the billed seat count with it. diff --git a/flake.nix b/flake.nix index f005644e3d..2a1d2699c8 100644 --- a/flake.nix +++ b/flake.nix @@ -78,15 +78,16 @@ ]; # --------------------------------------------------------------- - # Prebuilt V8 binary (must match version in Cargo.toml) + # Prebuilt V8 binary (must match the `v8` pin in backend/Cargo.toml). + # The simdutf variant: deno_core enables that feature of the v8 crate. # --------------------------------------------------------------- rustyV8Archive = let - version = "137.1.0"; + version = "150.4.0"; target = stdenv.hostPlatform.rust.rustcTarget; sha256 = { x86_64-linux = - "sha256-Tiscfy2bzYGR3s0T+SC1IB3xWvTVpVcSEdjq3MCRoRw="; + "sha256-9IdiyhDR8fxgWkQcWuQw7Izh6egPFNePvELLh4wwtHY="; aarch64-linux = lib.fakeHash; x86_64-darwin = lib.fakeHash; aarch64-darwin = lib.fakeHash; @@ -94,7 +95,7 @@ in pkgs.fetchurl { name = "librusty_v8-${version}"; url = - "https://github.com/denoland/rusty_v8/releases/download/v${version}/librusty_v8_release_${target}.a.gz"; + "https://github.com/denoland/rusty_v8/releases/download/v${version}/librusty_v8_simdutf_release_${target}.a.gz"; inherit sha256; }; diff --git a/frontend/e2e/global-setup.ts b/frontend/e2e/global-setup.ts index db27741a5a..6d10c53d92 100644 --- a/frontend/e2e/global-setup.ts +++ b/frontend/e2e/global-setup.ts @@ -47,18 +47,10 @@ async function globalSetup(config: FullConfig) { await page.waitForURL('**/') if (!process.env.SKIP_HUB_SYNC) { - await page.locator('#home-search-input').waitFor({ state: 'visible' }) - await page.locator('#home-search-input').fill('u/admin/hub_sync') - - const hubSyncBtn = page.locator('text=u/admin/hub_sync').first() - await hubSyncBtn.waitFor({ state: 'visible' }) - await hubSyncBtn.click() - - const runBtn = page.locator('#run-form-run-button') - await runBtn.waitFor({ state: 'visible' }) - await runBtn.click() - await page.waitForURL(/\/run\/.+/) - await page.locator('text=Success').waitFor({ timeout: 30000, state: 'visible' }) + const res = await page.request.post(`${baseURL}/api/settings/sync_cached_resource_types`) + if (!res.ok()) { + throw new Error(`Resource type sync failed: ${await res.text()}`) + } } // Save the authenticated state await context.storageState({ path: './e2e/auth.json' }) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 28ecbebf42..c8250f9eca 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.828.0", + "version": "1.830.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.828.0", + "version": "1.830.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index cc42f5424a..ddfc6b410b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.828.0", + "version": "1.830.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/apiBaseUrl.svelte.ts b/frontend/src/lib/apiBaseUrl.svelte.ts new file mode 100644 index 0000000000..242a6aa9e0 --- /dev/null +++ b/frontend/src/lib/apiBaseUrl.svelte.ts @@ -0,0 +1,21 @@ +import { fromStore } from 'svelte/store' +import { instanceUi } from '$lib/instanceUi' + +const ui = fromStore(instanceUi) + +/** + * Base of every URL shown for an external client to call (webhooks, HTTP routes, push + * trigger endpoints, the CLI remote): the `api_base_url` instance setting, else the + * origin being browsed. Append `/api/...` to it. + * + * Not for URLs the editor itself calls or opens (captures, previews, OAuth popups, links + * to pages): those must stay on the browsing origin, where the session cookie lives. + * + * Reactive: the setting lands after page load, so read it inside a `$derived` or the + * template rather than caching the result. + */ +export function apiBaseUrl(): string { + const setting = ui.current?.api_base_url + const configured = typeof setting === 'string' ? setting.trim() : '' + return configured || window.location.origin +} diff --git a/frontend/src/lib/base.ts b/frontend/src/lib/base.ts deleted file mode 100644 index 0e8f2d71f6..0000000000 --- a/frontend/src/lib/base.ts +++ /dev/null @@ -1 +0,0 @@ -export let base: string = import.meta.env.VITE_BASE_URL ?? '' diff --git a/frontend/src/lib/components/AgentTrace.svelte b/frontend/src/lib/components/AgentTrace.svelte index 7f2a90e621..3d671f46c8 100644 --- a/frontend/src/lib/components/AgentTrace.svelte +++ b/frontend/src/lib/components/AgentTrace.svelte @@ -1,7 +1,6 @@
diff --git a/frontend/src/lib/components/CompareWorkspaces.svelte b/frontend/src/lib/components/CompareWorkspaces.svelte index 9696836dfd..18c765e13b 100644 --- a/frontend/src/lib/components/CompareWorkspaces.svelte +++ b/frontend/src/lib/components/CompareWorkspaces.svelte @@ -65,7 +65,6 @@ import WorkspaceDeployLayout from './WorkspaceDeployLayout.svelte' import DeploymentRequestPanel from './deploymentRequest/DeploymentRequestPanel.svelte' import { userStore } from '$lib/stores' - import { base } from '$lib/base' import CompareModeToggle, { type CompareMode } from './CompareModeToggle.svelte' import CompareTargetPicker from './CompareTargetPicker.svelte' import { displayDate } from '$lib/utils' @@ -1461,7 +1460,7 @@ {test.test_script_path} {#if test.job_id} view run diff --git a/frontend/src/lib/components/DefaultScriptsInner.svelte b/frontend/src/lib/components/DefaultScriptsInner.svelte index b062fb4c49..48c7a8d9f7 100644 --- a/frontend/src/lib/components/DefaultScriptsInner.svelte +++ b/frontend/src/lib/components/DefaultScriptsInner.svelte @@ -16,11 +16,13 @@ let { small = false }: Props = $props() function computeLangs(defaultScripts: WorkspaceDefaultScripts | undefined): Script['language'][] { - const allLangs = Object.keys(defaultScriptLanguages) as Script['language'][] + const allLangs = (Object.keys(defaultScriptLanguages) as Script['language'][]).filter( + (l) => l != 'deno' + ) if (!defaultScripts || defaultScripts.order == undefined) return allLangs return defaultScripts.order ?.concat(allLangs.filter((l) => !defaultScripts.order?.includes(l))) - .filter((x) => x != 'nativets') as Script['language'][] + .filter((x) => x != 'nativets' && x != 'deno') as Script['language'][] } async function changePosition(i: number, up: boolean) { diff --git a/frontend/src/lib/components/DeployWorkspace.svelte b/frontend/src/lib/components/DeployWorkspace.svelte index 53ca12cf03..7913240188 100644 --- a/frontend/src/lib/components/DeployWorkspace.svelte +++ b/frontend/src/lib/components/DeployWorkspace.svelte @@ -5,7 +5,6 @@ aiAgentModuleDependencies, collectTransformRefs } from './deployDependencies' - import { base } from '$lib/base' import { enterpriseLicense, superadmin } from '$lib/stores' import { AppService, @@ -538,7 +537,7 @@ > {:else} Share this link to have another properly permissioned + >Share this link to have another properly permissioned user do the deployment @@ -580,7 +579,7 @@ connection={remoteStatus?.connection} returnTo={kind === 'trigger' ? undefined - : `${base}/deploy/${kind}/${initialPath}?workspace=${encodeURIComponent($operatingWorkspace!)}`} + : `/deploy/${kind}/${initialPath}?workspace=${encodeURIComponent($operatingWorkspace!)}`} onChange={loadDestinations} /> diff --git a/frontend/src/lib/components/DevWorkspaceSetting.svelte b/frontend/src/lib/components/DevWorkspaceSetting.svelte index ff9932fa87..5346cab7b0 100644 --- a/frontend/src/lib/components/DevWorkspaceSetting.svelte +++ b/frontend/src/lib/components/DevWorkspaceSetting.svelte @@ -8,7 +8,6 @@ import { sendUserToast } from '$lib/toast' import { switchWorkspace } from '$lib/storeUtils' import { goto } from '$app/navigation' - import { base } from '$lib/base' import { findCanonicalDevWorkspace, findWorkspaceAncestors, @@ -239,7 +238,7 @@ // scratch rather than from the current query so no stale `?workspace=` survives a switch. function rulesetsHref(name?: string): string { const rule = name ? `&rule=${encodeURIComponent(name)}` : '' - return `${base}/workspace_settings?tab=rulesets${rule}` + return `/workspace_settings?tab=rulesets${rule}` } function openRulesets(name?: string) { @@ -531,7 +530,7 @@ diff --git a/frontend/src/lib/components/DisplayResult.svelte b/frontend/src/lib/components/DisplayResult.svelte index a49f09bb02..afadcb5741 100644 --- a/frontend/src/lib/components/DisplayResult.svelte +++ b/frontend/src/lib/components/DisplayResult.svelte @@ -7,7 +7,6 @@ import { parseDbtRun } from '$lib/components/dbt/parseDbtRun' import { copyToClipboard, parseS3Object, roughSizeOfObject } from '$lib/utils' import ExpandableImage from '$lib/components/common/image/ExpandableImage.svelte' - import { base } from '$lib/base' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' import { appendViewToken } from '$lib/viewToken' import { Badge, Button, Drawer, DrawerContent } from './common' @@ -254,7 +253,7 @@ ) let resultDownloadHref = $derived( resultApiPath - ? `${base}/api${resultApiPath}` + ? `/api${resultApiPath}` : `data:text/json;charset=utf-8,${encodeURIComponent(toJsonStr(result))}` ) let resultDownloadName = $derived(`${filename ?? 'result'}.json`) @@ -856,7 +855,6 @@ {workspaceId} {jobId} {nodeId} - {base} {result} {disableTooltips} on:open-drawer={() => openDrawer()} diff --git a/frontend/src/lib/components/DisplayResultControlBar.svelte b/frontend/src/lib/components/DisplayResultControlBar.svelte index 45011d2db1..6af502a1b8 100644 --- a/frontend/src/lib/components/DisplayResultControlBar.svelte +++ b/frontend/src/lib/components/DisplayResultControlBar.svelte @@ -14,7 +14,6 @@ workspaceId?: string | undefined jobId?: string | undefined nodeId?: string | undefined - base: string result: any disableTooltips?: boolean } @@ -25,7 +24,6 @@ workspaceId = undefined, jobId = undefined, nodeId = undefined, - base, result, disableTooltips = false }: Props = $props() @@ -67,7 +65,7 @@ download={downloadName} class="text-current" href={resultApiPath - ? `${base}/api${resultApiPath}` + ? `/api${resultApiPath}` : `data:text/json;charset=utf-8,${encodeURIComponent(toJsonStr(result))}`} > diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 2db471e2c8..429c99c4a6 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -22,7 +22,6 @@ + + (opened = false)}> + +
+ {#snippet action()} + + {#snippet children({ item })} + {#each Object.keys(WINDOWS) as key (key)} + + {/each} + {/snippet} + + {/snippet} + +
+
+ Sort by + orderBy, + (v) => { + orderBy = v + logFeatureUsage('worker_resource_stats', 'sort', { key: v }) + } + } + noWFull + > + {#snippet children({ item })} + {#each Object.entries(ORDERS) as [value, label] (value)} + + {/each} + {/snippet} + +
+
+ +
{/if} {#if displayedFileKeys.length === 0} {#if fileListLoading} -
-
- Loading content -
+
+ + Loading...
{:else} -
+
{#if filter.trim() !== ''} No files starting with "{filter.trim()}" {:else} @@ -1131,19 +1244,19 @@ can remove every match. Without this the only control that could resume the listing would be hidden behind the empty state. -->
-
{/if} {/if} {:else} -
+
{#snippet header()}{/snippet} {#snippet footer()}{/snippet} @@ -1155,199 +1268,249 @@ {@const load_more_prefix = is_load_more ? item_key.slice(0, -1) : ''} {@const file_info = allFilesByKey[item_key]} -
+
{#if is_load_more} {@const loadMoreNesting = nestingLevelOf(load_more_prefix + 'x') - 2 * rootPathNestingLevel} {@const loadingMore = folderState[load_more_prefix]?.loading === true} - - -
!loadingMore && loadMore(load_more_prefix)} - class={twMerge( - 'flex flex-row h-full text-xs items-center justify-start text-secondary', - loadingMore ? 'cursor-default' : 'cursor-pointer' - )} + disabled={loadingMore} + class="w-full text-xs font-normal flex gap-2 items-center h-8 pr-1 text-secondary hover:bg-surface-hover cursor-pointer disabled:cursor-default disabled:hover:bg-transparent" + style={treeRowIndent(loadMoreNesting)} > -
- -
- {#if loadingMore} - - {:else} - - {/if} -
-
Load more
-
-
+ + {#if loadingMore} + + {:else} + + {/if} + Load more + {:else if file_info} {@const nestingLevel = file_info.nestingLevel - 2 * rootPathNestingLevel} - - -
selectItem(index)} + title={file_info.full_key.slice(rootPath.length)} class={twMerge( - 'flex flex-row h-full font-semibold text-xs items-center justify-start', - selectedFileKey !== undefined && selectedFileKey.s3 === file_info.full_key - ? 'bg-surface-hover' - : '' + 'w-full text-xs font-normal text-primary flex gap-2 items-center h-8 pr-1 cursor-pointer', + isSelected ? 'bg-surface-secondary' : 'hover:bg-surface-hover' )} + style={treeRowIndent(nestingLevel)} > -
- {#if file_info.type === 'folder'} - {#if folderState[file_info.full_key]?.loading} - - {:else if file_info.collapsed}{:else}{/if} -
- - {#if file_info.display_name === ''} - (empty name) - {:else}{file_info.display_name}{/if} - {#if !lazyMode} - ({file_info.count}{count % 1000 === 0 && - lastKeyFolders[file_info.nestingLevel / 2] === file_info.display_name - ? '+' - : ''} item{file_info.count === 1 ? '' : 's'}) - {/if} -
+ {#if isFolder} + {#if folderState[file_info.full_key]?.loading} + {:else} - -
- {file_info.display_name} -
+ {/if} -
-
+ {:else} + + {/if} + + + + {#if file_info.display_name === ''} + (empty name) + {:else}{file_info.display_name}{/if} + + + {#if isFolder && !lazyMode} + + {file_info.count}{count % 1000 === 0 && + lastKeyFolders[file_info.nestingLevel / 2] === file_info.display_name + ? '+' + : ''} + + {/if} + {/if}
{/snippet}
{#if fileListLoading === true} -
- Loading content +
+ Loading...
{:else if lazyMode}
{displayedCount} item{displayedCount === 1 ? '' : 's'} shown
{:else} -
+
{displayedCount}{flatHasMore ? '+' : ''} {displayedCount !== count ? 'filtered ' : ''}items (including inside folders)
{#if flatHasMore} - {/if} {/if}
{/if} -
+ {/if} -
- {#if fileMetadata === undefined} -
- {#if fileInfoLoading} -
- {:else if fileListUnavailable} -
- {:else} -
- {/if} -
- {:else} -
-
-

- {((p) => (p.startsWith(rootPath) ? p.slice(rootPath.length) : p))( - fileMetadata.fileKey - )} -

- {#if filePreview !== undefined && (!hideS3SpecificDetails || !readOnlyMode || allowDelete)} -
- {#if !hideS3SpecificDetails} - {@const downloadApiPath = `/w/${ws}/job_helpers/download_s3_file?file_key=${encodeURIComponent(fileMetadata?.fileKey ?? '')}${storage ? `&storage=${storage}` : ''}${s3ResourcePath ? `&s3_resource_path=${encodeURIComponent(s3ResourcePath)}` : ''}`} - {@const downloadName = - fileMetadata?.fileKey.split('/').pop() ?? 'unnamed_download.file'} - {#if shouldDownloadViaClient()} -
+ {:else if !csvOptions} +
+ {#if filePreview.contentType === 'Csv'} + Previewing a {filePreview.contentType?.toLowerCase()} file. Separator character: +
+ +
+ Header row: +
+ +
+ {:else} + Previewing a {filePreview.contentType?.toLowerCase()} file. + {/if} +
+ {/if} 
{#if !emptyString(filePreview.contentPreview)}{filePreview.contentPreview}{:else if filePreview.contentType !== undefined}Preview impossible.{/if}
diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index f69cf37f1d..601bf77799 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -72,7 +72,6 @@ Tag, X } from 'lucide-svelte' - import { base } from '$lib/base' import { useLocalStorageValue } from '$lib/svelte5Utils.svelte' import { parsePipelineAnnotations } from './assets/AssetGraph/parsePipelineAnnotations' import DropdownV2 from './DropdownV2.svelte' @@ -128,8 +127,9 @@ import WacExportDrawer from './scripts/WacExportDrawer.svelte' import { UserDraft } from '$lib/userDraft.svelte' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' - import { useTriggerLock } from '$lib/operatorWriteRights' + import { useEditRights, useTriggerLock } from '$lib/operatorWriteRights' const triggerLock = useTriggerLock() + const editRights = useEditRights() let { script = $bindable(), @@ -1385,7 +1385,7 @@ onCanceled={() => (perpetualRunsToConfirm = undefined)} /> -{#if !actingUser?.operator} +{#if editRights.roleCanAuthor('script')} to materialize its result, or build it in the - pipeline editor. + pipeline editor. Enable Windmill AI in the diff --git a/frontend/src/lib/components/ScriptPicker.svelte b/frontend/src/lib/components/ScriptPicker.svelte index 781cb5155e..97ac81713d 100644 --- a/frontend/src/lib/components/ScriptPicker.svelte +++ b/frontend/src/lib/components/ScriptPicker.svelte @@ -1,7 +1,6 @@ diff --git a/frontend/src/lib/components/common/confirmationModal/OtherUsersDraftsModal.svelte b/frontend/src/lib/components/common/confirmationModal/OtherUsersDraftsModal.svelte index 49356d888e..b6ec78f336 100644 --- a/frontend/src/lib/components/common/confirmationModal/OtherUsersDraftsModal.svelte +++ b/frontend/src/lib/components/common/confirmationModal/OtherUsersDraftsModal.svelte @@ -7,7 +7,8 @@ */ import { DraftService, type UserDraftItemKind } from '$lib/gen' import { sendUserToast } from '$lib/toast' - import { Users, Pencil, GitCompareArrows, Wrench } from 'lucide-svelte' + import { Users, Pencil, GitCompareArrows, Wrench, Trash2 } from 'lucide-svelte' + import ConfirmationModal from './ConfirmationModal.svelte' import Modal2 from '$lib/components/common/modal/Modal2.svelte' import Button from '$lib/components/common/button/Button.svelte' import Tooltip from '$lib/components/Tooltip.svelte' @@ -17,6 +18,10 @@ import { OtherUserDraftLoad } from '$lib/components/otherUserDraftLoad.svelte' import { displayDate } from '$lib/utils' import { userStore } from '$lib/stores' + import { armRestartOnFirstInteraction } from '$lib/userDraftToast' + import { UserDraft } from '$lib/userDraft.svelte' + import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' + import { goto } from '$app/navigation' export type OtherDraftUser = { username?: string | null; draft_saved_at?: string } @@ -54,8 +59,70 @@ let diffDrawer: DiffDrawer | undefined = $state(undefined) let migrateOpen = $state(false) - // Legacy (no-owner) drafts can only be resolved by workspace admins / superadmins. - const canMigrateLegacy = $derived(!!$userStore?.is_admin || !!$userStore?.is_super_admin) + // One owner's draft, or every draft at the path. + let deleting = $state(undefined) + let deleteBusy = $state(false) + + // Legacy (no-owner) drafts can only be resolved by workspace admins / superadmins, + // and only they can delete a draft that belongs to someone else. + const isAdmin = $derived(!!$userStore?.is_admin || !!$userStore?.is_super_admin) + + async function deleteDraft(target: OtherDraftUser | 'all') { + const username = target === 'all' ? undefined : target.username + if (target !== 'all' && !username) return + deleteBusy = true + const query = { workspace, itemKind, path } + try { + // A path-wide delete takes our own draft too. Mute autosave and let any save + // already in flight land first: one committing after the delete would put the + // draft back for the reload below to read. + if (target === 'all') { + UserDraft.stopSync(itemKind, path, { workspace }) + await UserDraftDbSyncer.quiesce(query) + } + await DraftService.deleteDraftForUser({ + workspace, + kind: itemKind, + path, + username: username ?? undefined + }) + } catch (e) { + if (target === 'all') { + UserDraft.restartSync(itemKind, path, { workspace }) + // `quiesce` cancelled the timer of an edit still waiting to save. + void UserDraftDbSyncer.flush(query, { honorAutosaveToggle: true }) + } + sendUserToast(`Could not delete draft: ${e.body ?? e.message}`, true) + return + } finally { + deleteBusy = false + } + sendUserToast(username ? `Deleted ${username}'s draft` : `Deleted all drafts at ${path}`) + deleting = undefined + isOpen = false + + // Drop the editor's copy of our draft as well, or its next write would save + // it right back. Autosave re-arms on the next interaction. + if (target === 'all') { + UserDraft.remove(itemKind, path, { workspace }) + await UserDraftDbSyncer.flush(query) + armRestartOnFirstInteraction(workspace, itemKind, path) + } + // Nothing is left at a never-deployed path once its last draft is gone, so + // there is no item to reload the editor on. + const lastDraftGone = + target === 'all' || + (!hasOwnDraft && otherDraftsUsers.every((o) => ownerKey(o) === ownerKey(target))) + if (draftOnly && lastDraftGone) { + await goto('/') + return + } + try { + await onReload?.() + } catch (e) { + sendUserToast(`Could not reload the editor: ${e.body ?? e.message}`, true) + } + } function ownerLabel(owner: OtherDraftUser): string { return owner.username ?? 'Legacy draft' @@ -184,7 +251,7 @@ > Load - {#if !owner.username && canMigrateLegacy} + {#if !owner.username && isAdmin} + {:else if owner.username && isAdmin} + {/if} {/each} -
+
+ {#if isAdmin} + + {/if}
@@ -206,6 +296,29 @@ + { + if (deleting) void deleteDraft(deleting) + }} + onCanceled={() => (deleting = undefined)} +> + + {#if deleting === 'all'} + This permanently deletes every draft at + {path}, yours included. All unsaved work there + is lost and can't be recovered. + {:else} + This permanently deletes the draft {deleting?.username} has at + {path}. Their unsaved work there is lost and + can't be recovered. + {/if} + + + import { Download } from 'lucide-svelte' - import { base } from '$lib/base' import { downloadViaClient, shouldDownloadViaClient } from '$lib/utils/downloadFile' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' @@ -26,7 +25,7 @@ s3object?.storage ? `&storage=${s3object.storage}` : '' }${appPath && s3object?.presigned ? `&${s3object?.presigned}` : ''}` ) - let href = $derived(`${base}/api${apiPath}`) + let href = $derived(`/api${apiPath}`) const sharedClass = `relative center-center flex w-full text-center font-normal text-primary text-sm border border-dashed border-gray-400 hover:border-blue-500 diff --git a/frontend/src/lib/components/common/slack/SlackConnectionStatus.svelte b/frontend/src/lib/components/common/slack/SlackConnectionStatus.svelte index 8658c462ab..d79491b0fc 100644 --- a/frontend/src/lib/components/common/slack/SlackConnectionStatus.svelte +++ b/frontend/src/lib/components/common/slack/SlackConnectionStatus.svelte @@ -1,7 +1,6 @@ diff --git a/frontend/src/lib/components/common/table/AgentRow.svelte b/frontend/src/lib/components/common/table/AgentRow.svelte index ecde985be4..aca6ae7f9c 100644 --- a/frontend/src/lib/components/common/table/AgentRow.svelte +++ b/frontend/src/lib/components/common/table/AgentRow.svelte @@ -1,5 +1,4 @@ {#snippet chatBadge()} @@ -69,6 +71,7 @@ workspaceId={agent.workspace_id ?? $workspaceStore ?? ''} canFavorite={false} {depth} + {rowSelection} > {#snippet badges()} @@ -99,8 +102,9 @@ {/if} - agentMenuItems({ + items={async () => [ + ...selectMenuItems(rowSelection), + ...agentMenuItems({ deployUiSettings: await getDeployUiSettings(), path: agent.path, canWrite: agent.canWrite, @@ -113,7 +117,8 @@ if (event?.shiftKey) remove(path) else deleteConfirmedCallback = () => remove(path) } - })} + }) + ]} on:open={() => { menuOpen = true }} diff --git a/frontend/src/lib/components/common/table/AppRow.svelte b/frontend/src/lib/components/common/table/AppRow.svelte index bd881a31b7..a318c1f412 100644 --- a/frontend/src/lib/components/common/table/AppRow.svelte +++ b/frontend/src/lib/components/common/table/AppRow.svelte @@ -1,5 +1,4 @@ {#if menuOpen} @@ -141,8 +140,8 @@ aiId={`flow-row-${flow.path}`} aiDescription={`Button to access the form to run the flow ${flow.summary ?? flow.path}`} href={flow.draft_only - ? `${base}/flows/edit/${flow.path}` - : `${base}/flows/get/${flow.path}?workspace=${$workspaceStore}`} + ? `/flows/edit/${flow.path}` + : `/flows/get/${flow.path}?workspace=${$workspaceStore}`} kind="flow" workspaceId={flow.workspace_id ?? $workspaceStore ?? ''} {keyboardSelected} @@ -200,7 +199,7 @@ wrapperClasses="w-20" unifiedSize="md" startIcon={{ icon: Pen }} - href="{base}/flows/edit/{flow.path}" + href="/flows/edit/{flow.path}" aiId={`edit-flow-button-${flow.summary?.length > 0 ? flow.summary : flow.path}`} aiDescription={`Edits the flow ${flow.summary?.length > 0 ? flow.summary : flow.path}`} > @@ -264,12 +263,12 @@ { displayName: 'View runs', icon: List, - href: `${base}/runs/${path}` + href: `/runs/${path}` }, { displayName: 'Duplicate/Fork', icon: GitFork, - href: `${base}/flows/add?template=${path}`, + href: `/flows/add?template=${path}`, disabled: !showEditButton, hide: hideForOperator }, @@ -287,7 +286,7 @@ { displayName: 'Audit logs', icon: Eye, - href: `${base}/audit_logs?resource=${path}`, + href: `/audit_logs?resource=${path}`, hide: $userStore?.operator }, { diff --git a/frontend/src/lib/components/common/table/PipelineRow.svelte b/frontend/src/lib/components/common/table/PipelineRow.svelte index 509297442b..334c8fb5fd 100644 --- a/frontend/src/lib/components/common/table/PipelineRow.svelte +++ b/frontend/src/lib/components/common/table/PipelineRow.svelte @@ -1,5 +1,4 @@ import { goto } from '$lib/navigation' - import { base } from '$lib/base' import Dropdown from '$lib/components/DropdownV2.svelte' import type MoveDrawer from '$lib/components/MoveDrawer.svelte' import ScheduleEditor from '$lib/components/triggers/schedules/ScheduleEditor.svelte' @@ -157,8 +156,8 @@ aiId={`script-run-button-${script.path}`} aiDescription={`Button to access the form to run the script ${script.summary ?? script.path}`} href={script.draft_only || (script.auto_kind === 'lib' && script.kind !== 'preprocessor') - ? `${base}/scripts/edit/${script.path}` - : `${base}/scripts/get/${script.hash}?workspace=${$workspaceStore}`} + ? `/scripts/edit/${script.path}` + : `/scripts/get/${script.hash}?workspace=${$workspaceStore}`} kind="script" {keyboardSelected} {marked} @@ -261,7 +260,7 @@ wrapperClasses="w-20" unifiedSize="md" startIcon={{ icon: Pen }} - href="{base}/scripts/edit/{script.path}" + href="/scripts/edit/{script.path}" > Edit @@ -351,7 +350,7 @@ { displayName: 'Duplicate/Fork', icon: GitFork, - href: `${base}/scripts/add?template=${script.path}`, + href: `/scripts/add?template=${script.path}`, disabled: !showEditButton, hide: $userStore?.operator }, @@ -391,7 +390,7 @@ { displayName: 'View runs', icon: List, - href: `${base}/runs/${script.path}` + href: `/runs/${script.path}` }, { displayName: 'Versions', @@ -403,7 +402,7 @@ { displayName: 'Audit logs', icon: Eye, - href: `${base}/audit_logs?resource=${script.path}`, + href: `/audit_logs?resource=${script.path}`, hide: $userStore?.operator }, { diff --git a/frontend/src/lib/components/common/teams/TeamsConnectionStatus.svelte b/frontend/src/lib/components/common/teams/TeamsConnectionStatus.svelte index ce368e93b9..64b9269f3d 100644 --- a/frontend/src/lib/components/common/teams/TeamsConnectionStatus.svelte +++ b/frontend/src/lib/components/common/teams/TeamsConnectionStatus.svelte @@ -1,7 +1,6 @@ diff --git a/frontend/src/lib/components/copilot/AIFormAssistant.svelte b/frontend/src/lib/components/copilot/AIFormAssistant.svelte index b4526f1153..48a46ad182 100644 --- a/frontend/src/lib/components/copilot/AIFormAssistant.svelte +++ b/frontend/src/lib/components/copilot/AIFormAssistant.svelte @@ -8,7 +8,6 @@ import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' import { copilotInfo } from '$lib/aiStore' import { logFeatureUsage } from '$lib/utils/featureUsage' - import { pageHref } from '$lib/components/sessions/previewPaths' interface Props { /** Unset for a viewer who cannot edit the item: the card then offers no way to the editor. */ @@ -45,7 +44,7 @@ const sessionSource = $derived( path ? { - page: () => pageHref(`/${runnableType}s/get/${path}`), + page: () => `/${runnableType}s/get/${path}`, workspaceId: $operatingWorkspace ?? undefined, beforeOpen: logAsked } diff --git a/frontend/src/lib/components/copilot/CronGen.svelte b/frontend/src/lib/components/copilot/CronGen.svelte index 94715a0f18..fb5806234c 100644 --- a/frontend/src/lib/components/copilot/CronGen.svelte +++ b/frontend/src/lib/components/copilot/CronGen.svelte @@ -5,7 +5,6 @@ import Popover from '$lib/components/meltComponents/Popover.svelte' import { sendUserToast } from '$lib/toast' - import { base } from '$lib/base' import type { ChatCompletionMessageParam } from 'openai/resources/index.mjs' import { copilotInfo } from '$lib/aiStore' import { untrack } from 'svelte' @@ -132,7 +131,7 @@

Enable Windmill AI in the workspace settings

Enable Windmill AI in the diff --git a/frontend/src/lib/components/copilot/RegexGen.svelte b/frontend/src/lib/components/copilot/RegexGen.svelte index f399726988..32fdc4a5f1 100644 --- a/frontend/src/lib/components/copilot/RegexGen.svelte +++ b/frontend/src/lib/components/copilot/RegexGen.svelte @@ -1,7 +1,6 @@ @@ -416,7 +460,7 @@ {#snippet status()} - {#if !pending && !inspected} + {#if !pending && !inspected && statusTime} {statusTime} @@ -591,7 +635,7 @@ : 'No logs.'}

{/if} - {#if running && !inspected} + {#if running && !waiting && !inspected}
streaming diff --git a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte index 38f91353a2..832a96d6b4 100644 --- a/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte +++ b/frontend/src/lib/components/copilot/chat/ToolExecutionDisplay.svelte @@ -13,7 +13,6 @@ ExternalLink, BookOpen } from 'lucide-svelte' - import { base } from '$lib/base' import { truncateRev } from '$lib/utils' import { EXIT_PLAN_MODE_TOOL, @@ -301,7 +300,7 @@ {#snippet jobLink()} ({ base: '' })) - import { shareWorkspaceId } from './artifactSharing' describe('shareWorkspaceId', () => { diff --git a/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts index 57b08cba9b..269eb6354b 100644 --- a/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts +++ b/frontend/src/lib/components/copilot/chat/artifacts/artifactSharing.ts @@ -1,5 +1,3 @@ -import { base } from '$lib/base' - /** * The workspace a share from `workspaceId` lands in: the topmost ancestor the user still * belongs to. A session often runs in a fork, whose members are its creator alone, so a link @@ -20,7 +18,7 @@ export function shareWorkspaceId( } export function sharedArtifactUrl(workspaceId: string, shareId: string): string { - return `${window.location.origin}${base}/shared_artifacts/${encodeURIComponent( + return `${window.location.origin}/shared_artifacts/${encodeURIComponent( shareId )}?workspace=${encodeURIComponent(workspaceId)}` } diff --git a/frontend/src/lib/components/copilot/chat/global/core.ts b/frontend/src/lib/components/copilot/chat/global/core.ts index b04493355e..2a795f1f81 100644 --- a/frontend/src/lib/components/copilot/chat/global/core.ts +++ b/frontend/src/lib/components/copilot/chat/global/core.ts @@ -26,7 +26,7 @@ import { import { createTwoFilesPatch } from 'diff' import { deepEqual } from 'fast-equals' import { promptSafe, type ArtifactVersionTarget } from '$lib/components/sessions/previewRouter' -import { canWrite } from '$lib/utils' +import { canDraftItem, roleCanDraft } from '$lib/editRights' import { $ScriptLang } from '$lib/gen/schemas.gen' import type { AppWithLastVersion, @@ -75,6 +75,8 @@ import { formatDataTableRef } from '$lib/components/raw_apps/dataTableRefUtils' import { appSourceToDraftValue } from '$lib/components/raw_apps/rawAppDraftValue' +import { conformBuilderAppPolicy } from '$lib/components/raw_apps/builderAppPolicy' +import { operatorBuilderAppsIn } from '$lib/operatorWriteRights' import type { RawAppDomQuery } from '$lib/components/raw_apps/rawAppDom' import { dataUrlToImagePart, normalizeImageDataUrl, type AttachedImage } from '../imageUtils' import { sanitizeAttachmentName, textLineCount, type AttachedTextFile } from '../textFileUtils' @@ -212,10 +214,13 @@ import { invalidateWorkspaceComparison } from '$lib/workspaceComparison' import type { UserDraftItemKind } from '$lib/gen' import { bundleRawAppDraft } from './rawAppBundlerBridge' import { + canOpenEditor, DEPLOY, NONE, + RUN_FLOW_PREVIEW, RUN_PREVIEW, WRITE_DRAFT, + WRITE_FLOW_DRAFT, type SessionAccess, type SessionTool, type SessionToolPolicy @@ -241,7 +246,6 @@ import { parseItemsMaskParam } from '$lib/components/sessions/modifiedItemsMask' import { - pageHref, TRIGGER_PAGES, type TriggerKind as PageTriggerKind } from '$lib/components/sessions/previewRouter' @@ -331,7 +335,7 @@ export type GlobalActiveEditorContext = { export type GlobalActivePreviewContext = { /** Page name as the tab strip shows it, e.g. "Schedules". */ label: string - /** Base-stripped page path plus the request params the page declares — values kept + /** Page path plus the request params the page declares — values kept * only for the ones addressing a workspace object, and percent-encoded. Never a raw * location: a tab can host a legacy app whose hash is app state, and a filter value * can be free text the user typed. Build it with `previewLocationContext`. */ @@ -763,7 +767,7 @@ const listRunsSchema = z.object({ // superadmin caller; every other caller gets the hedge even where nothing is hidden. const NO_WORKERS_VISIBLE_MESSAGE = 'No workers came back. This does NOT establish that no workers are running: an instance can hide workers from callers without the devops role, and it does so by returning an empty list rather than an error. ' + - 'Tell the user you cannot see any workers and that worker visibility may be restricted for your account, and suggest they check the Workers page themselves. Never state that no workers are online or that the instance has none.' + 'Tell the user you cannot see any workers and that worker visibility may be restricted for your account, and suggest they check the Workers page themselves if they can open it, or ask a workspace admin otherwise. Never state that no workers are online or that the instance has none.' const NO_WORKERS_CONNECTED_MESSAGE = 'No workers are connected to this instance (none pinged in the last 5 minutes). Queued runs will stay queued until a worker starts.' const WORKER_PAGE_SIZE = 100 @@ -1184,7 +1188,14 @@ const openPreviewSchema = z.object({ * it can open, so it is not offered. Per-PATH write permission is unknowable here — * a workspace capability says nothing about `f/finance/*` — and is resolved in the * handler instead. Mirrors `open_page` re-narrowing its page enum per user. */ +const openPreviewNoPipelineKind = z + .enum(['script', 'flow', 'raw_app']) + .describe( + 'Item kind to preview. Use "raw_app" for code-based apps. The legacy drag-and-drop app builder ("app") is not previewable in the session panel — don\'t pass it.' + ) + const openPreviewViewOnlySchema = openPreviewSchema.extend({ + kind: openPreviewNoPipelineKind, mode: z .literal('view') .optional() @@ -1193,6 +1204,17 @@ const openPreviewViewOnlySchema = openPreviewSchema.extend({ ) }) +/** A builder operator: the flow editor, and the deployed page for everything else. */ +const openPreviewFlowEditorSchema = openPreviewSchema.extend({ + kind: openPreviewNoPipelineKind, + mode: z + .enum(['edit', 'view']) + .optional() + .describe( + 'Which side to show. "edit" opens the flow editor and works for flows only: the only editor you can open in this workspace. Pass "view" for a script or app, the deployed page with its run form, triggers and past runs.' + ) +}) + const OPEN_PREVIEW_DESCRIPTION = 'Open the live preview / editor for a workspace item in the side panel next to the chat. ONLY works inside an AI session — call this after writing or editing a script, flow, or raw app to let the user see and interact with it. The path you pass is the path of the item; for code-based apps use kind="raw_app" (legacy drag-and-drop apps are not previewable). Returns an error if there is no active session.' @@ -1407,7 +1429,14 @@ const buildGlobalSystemPrompt = ( // Each `can*` mirrors the capability the matching tools declare in `requires`, so a // rule cannot outlive the tool it describes. An unresolved profile keeps every block. const canWriteDraft = !access || access.has('write_draft') + const canWriteFlowDraft = !access || access.has('write_flow_draft') + // A builder operator: flow drafts and nothing else. Blocks about drafting in general + // follow `canWriteAnyDraft`; blocks naming scripts, apps or the other kinds keep `canWriteDraft`. + const flowsOnly = canWriteFlowDraft && !canWriteDraft + const canWriteAnyDraft = canWriteDraft || canWriteFlowDraft const canRunPreview = !access || access.has('run_preview') + const canManageSchedules = !access || access.has('manage_schedules') + const canManageTriggers = !access || access.has('manage_triggers') // The deploy tools take their kind as an argument, so `deploy` gates only the folder text. const canCreateFolder = !access || access.has('deploy') // Each gated block carries its own leading newline, so dropping one leaves no blank @@ -1448,9 +1477,12 @@ The current user's workspace username is "${username}".${instanceLine} ${ canWriteDraft ? 'Use tools to inspect workspace items and create per-user drafts (saved server-side, visible only to this user — not deployed) for scripts, flows, schedules, triggers, resources, variables, and raw apps.' - : "Use tools to inspect workspace items and the workspace's run history, and to run items that are already deployed. You cannot create or edit scripts, flows, apps, schedules, triggers, resources or variables here — this user's role does not allow it — so when they ask for such a change, say plainly that you cannot make it rather than describing steps as if you had. Their role is refused scripts, flows and apps outside this chat too, so for those suggest asking a workspace admin rather than creating them in the editor." + : `Use tools to inspect workspace items and the workspace's run history, and to run items that are already deployed. You cannot create or edit ${flowsOnly ? '' : 'flows, '}scripts, apps, schedules, triggers, resources or variables here — this user's role does not allow it — so when they ask for such a change, say plainly that you cannot make it rather than describing steps as if you had. Their role is refused ${flowsOnly ? 'scripts and apps' : 'scripts, flows and apps'} outside this chat too, so for those suggest asking a workspace admin rather than creating them in the editor. Resources and variables they can still edit themselves on their pages, so when open_page lists that page, open it for them instead.${when( + flowsOnly, + '\n\nFlows are the exception: this workspace lets them compose flows out of scripts and flows that are already deployed. Write flow drafts with write_flow and patch_flow_json, using only steps that call a deployed script or flow by path. The server refuses inline code anywhere in the flow, so never add a rawscript step or inline code; when the flow needs logic no deployed script provides, say so and suggest asking a workspace admin for that script.' + )}` }${when( - canWriteDraft, + canWriteAnyDraft, ` Path conventions: @@ -1463,25 +1495,35 @@ Path conventions: '; create one with `create_folder` only when the user explicitly asks for a new folder' )}.${folderGuidanceBlock}` )}${when( - canCreateFolder && !canWriteDraft, + canCreateFolder && !canWriteAnyDraft, '\n- You can create a shared folder with `create_folder` when the user explicitly asks for one. You cannot create anything inside it here, so do not offer to.' )} Rules:${when( - canWriteDraft, + canWriteAnyDraft, ` - Draft tools create or update drafts only; they do not deploy or mutate deployed workspace items.` )} - Use list_workspace_items to find items and read_workspace_item before changing an existing item. For triggers, pass trigger_kind. - If the user message includes an ACTIVE EDITOR section, treat it as the currently open item and use it for references like "this", "current", or "open editor".${activePreviewRule}${when( - canWriteDraft, + canWriteAnyDraft, ` - Use deploy_workspace_item only after the user explicitly asks to deploy. It persists a draft to the workspace. - To undo something you created or changed in this chat, use discard_local_draft: everything you write is a draft until it is explicitly deployed, so "delete it" / "never mind" / "remove that" about your own work means discarding the draft (it also clears the matching open editor draft). Use delete_workspace_item only to remove an item that is already deployed in the workspace; it mutates the workspace and fails if nothing is deployed at that path.` )}${when( - !canWriteDraft, + !canWriteAnyDraft, ` -- Three changes are still open to you where the server allows them: discard_local_draft drops a draft this user left behind, deploy_workspace_item deploys one when the user asks, and delete_workspace_item removes an item already deployed in the workspace. You cannot create or edit one.` +- Three changes are still open to you where the server allows them: discard_local_draft drops a draft this user left behind, deploy_workspace_item deploys one when the user asks, and delete_workspace_item removes an item already deployed in the workspace — each only for the kinds its type lists. You cannot create or edit one.` + )}${when( + !canManageSchedules || !canManageTriggers, + ` +- This workspace does not let this user change ${ + !canManageSchedules && !canManageTriggers + ? 'schedules or triggers' + : !canManageSchedules + ? 'schedules' + : 'triggers' + }, here or in the editor, so never ask them to create or edit one: suggest asking a workspace admin.` )} - Use diff to review changes — before deploying, or when the user asks what changed. It is read-only: without arguments it lists every draft in the workspace with its change status; with type+path it returns that item's unified diff (for multi-file apps, pass file to read one file's diff). In a fork, pass against="parent_workspace" to compare the deployed fork with its parent workspace instead. Pass search to grep changed lines across all diffs.${when( canWriteDraft, @@ -1499,6 +1541,10 @@ ${pipelineBullet}` ` - After creating or editing a script or flow draft, run test_run_script, test_run_flow, or test_run_step with representative args before reporting that it works. These tools prefer drafts, so testing does not require deployment. - Do the same for a raw app: run test_run_app_runnable on each backend runnable you wrote or changed before saying the app works. A bundle that compiles proves nothing about whether the runnables run. An inline runnable executes the app's draft code; a path runnable executes the DEPLOYED script/flow it names, so a path runnable aimed at something you have not deployed fails here — that failure is the point: report it and offer to deploy that one target. The app itself does not need deploying to be tested.` + )}${when( + flowsOnly && (!access || access.has('run_flow_preview')), + ` +- After creating or editing a flow draft, run test_run_flow with representative args before reporting that it works. It prefers drafts, so testing does not require deployment.` )} - Use list_runs to find recent runs (optionally filtered by path, creator, label, or status), then get_run with a returned id to see what that run was called with, what it returned and what it logged — without starting a new test run. - get_run also covers what a flow run did per step — statuses and results across the whole execution tree, subflow steps and loop iterations included — and works while the flow is still running. Pass step to read one step's result in full (capped at 12k chars). @@ -1523,7 +1569,11 @@ ${pipelineBullet}` ? `${when( !canWriteDraft, ` -- open_preview(kind, path) shows a deployed script / flow / app in the side panel next to the chat — its run form, triggers and past runs, which the user can act on there. Use it when you surface an item they will want to run or inspect. You cannot open item editors in this workspace, so it only ever opens the deployed page.` +- open_preview(kind, path) shows a deployed script / flow / app in the side panel next to the chat — its run form, triggers and past runs, which the user can act on there. Use it when you surface an item they will want to run or inspect. ${ + flowsOnly + ? 'The flow editor is the only editor you can open here: after writing or editing a flow draft, show it with open_preview(kind="flow", path, mode="edit").' + : 'You cannot open item editors in this workspace, so it only ever opens the deployed page.' + }` )}${when( canWriteDraft, ` @@ -1555,7 +1605,7 @@ Documentation: Flows: - read_workspace_item returns compact flow JSON. Inline script bodies appear as "inline_script.". - Use read_flow_module_code${when(canWriteDraft, ' and set_flow_module_code')} for inline script bodies.${when( - canWriteDraft, + canWriteFlowDraft, ` - Use patch_flow_json for structural flow edits and write_flow for full flow rewrites.` )} @@ -2020,6 +2070,11 @@ async function saveAppDraft( path: string, value: AppDraftValue ): Promise { + if (operatorBuilderAppsIn(workspace)) { + const role = await roleForWorkspace(workspace) + value.policy ??= {} + conformBuilderAppPolicy(value.policy, role.kind === 'resolved' ? role.user : undefined) + } return saveGlobalAppDraft(workspace, path, value) } @@ -2547,8 +2602,13 @@ export function getSessionContextPromptSection( // Concatenated onto an already capability-gated prompt, so it has to honour the same // profile rather than assume the gating happened upstream. const canDeploy = !access || access.has('deploy') - const canWriteDraft = !access || access.has('write_draft') - const canRunPreview = !access || access.has('run_preview') + const canWriteDraft = !access || access.has('write_draft') || access.has('write_flow_draft') + const canRunPreview = !access || access.has('run_preview') || access.has('run_flow_preview') + // The kinds no deploy rule reaches, unless the workspace withdrew them from this user. + const unruledKinds = [ + (!access || access.has('manage_schedules')) && 'schedules', + (!access || access.has('manage_triggers')) && 'triggers' + ].filter((k): k is string => !!k) const targets = ['reads', canWriteDraft && 'drafts', canRunPreview && 'test runs', 'deploys'] .filter(Boolean) .join(', ') @@ -2587,7 +2647,13 @@ export function getSessionContextPromptSection( // promotes the rest instead. The rules gate deletes too, so it is owed to every such profile. if (!canDeploy) { lines.push( - "- This workspace refuses direct deployment for this user, except for schedules and triggers — those are the only kinds deploy_workspace_item and delete_workspace_item can still act on. Scripts, flows, apps, resources and variables must be promoted from the session's deploy panel (fork or pull request); do not offer to deploy or delete them directly." + `- This workspace refuses direct deployment for this user${ + unruledKinds.length === 2 + ? ', except for schedules and triggers — those are the only kinds deploy_workspace_item and delete_workspace_item can still act on' + : unruledKinds.length === 1 + ? `, except for ${unruledKinds[0]} — the only kind deploy_workspace_item and delete_workspace_item can still act on` + : '' + }. Scripts, flows, apps, resources and variables must be promoted from the session's deploy panel (fork or pull request); do not offer to deploy or delete them directly.` ) } return lines.join('\n') @@ -3394,7 +3460,7 @@ export const openPageTool: SessionTool<{}> = { // is no active session, in which case we offer a link chip instead. const previewResult = openPagePreview({ sessionId: sessionIdFromCtx(ctx), - href: pageHref(url), + href: url, label: pageLabel, newTab: parsed.new_tab ?? false }) @@ -3431,9 +3497,14 @@ export const globalTools: SessionTool<{}>[] = [ // Every subject is authoring guidance written around the draft tools by name, and a // tool result is the one place the toolset filter cannot reach. const access = (ctx.helpers as GlobalToolHelpers | undefined)?.access - if (access && !access.has('write_draft')) { - const message = - 'This session cannot create or edit workspace items, so there is no authoring guidance to give. Tell the user plainly rather than describing how it would be done.' + const canAuthor = + !access || + access.has('write_draft') || + (parsed.subject === 'flow' && access.has('write_flow_draft')) + if (!canAuthor) { + const message = access?.has('write_flow_draft') + ? 'This session can only compose flows, so flow is the one subject with authoring guidance here. Tell the user plainly that you cannot create or edit this kind of item.' + : 'This session cannot create or edit workspace items, so there is no authoring guidance to give. Tell the user plainly rather than describing how it would be done.' toolCallbacks.setToolStatus(toolId, { content: 'No authoring guidance' }) return message } @@ -3781,7 +3852,7 @@ export const globalTools: SessionTool<{}>[] = [ } }, { - requires: WRITE_DRAFT, + requires: WRITE_FLOW_DRAFT, def: createToolDef(writeFlowSchema, 'write_flow', 'Create or overwrite a draft flow.'), showDetails: true, streamArguments: true, @@ -3915,7 +3986,7 @@ export const globalTools: SessionTool<{}>[] = [ } }, { - requires: WRITE_DRAFT, + requires: WRITE_FLOW_DRAFT, def: createToolDef( patchFlowJsonSchema, 'patch_flow_json', @@ -3965,7 +4036,7 @@ export const globalTools: SessionTool<{}>[] = [ autoCollapseDetails: false }, { - requires: RUN_PREVIEW, + requires: RUN_FLOW_PREVIEW, def: testRunFlowToolDef, fn: async (ctx) => { const parsed = testRunFlowSchema.parse(ctx.args) @@ -4145,16 +4216,17 @@ export const globalTools: SessionTool<{}>[] = [ // turns on the kind. Each kind's handler, reached through `deployDraft`'s switch: requires: NONE, kindRequires: { - // scripts.rs `create_script_internal`, flows.rs `create_flow`/`update_flow` and - // apps.rs `create_app_raw`/`update_app_raw` refuse operators; all run the rules. + // scripts.rs `create_script_internal` and apps.rs `create_app_raw`/`update_app_raw` + // refuse operators, flows.rs `create_flow`/`update_flow` those without builder + // rights; all run the rules. script: ['deploy', 'manage_code'], - flow: ['deploy', 'manage_code'], + flow: ['deploy', 'manage_flows'], app: ['deploy', 'manage_code'], resource: DEPLOY, variable: DEPLOY, - // The schedule and trigger handlers check neither. - schedule: NONE, - trigger: NONE + // No deploy rules here, but `gate_operator_writes` on their routers. + schedule: ['manage_schedules'], + trigger: ['manage_triggers'] } satisfies Record<(typeof ITEM_TYPES)[number], SessionToolPolicy>, def: createToolDef( deployWorkspaceItemSchema, @@ -4172,8 +4244,17 @@ export const globalTools: SessionTool<{}>[] = [ } }, { - // Rebasing writes a fresh draft, so drafts.rs `require_can_write_path` applies. - requires: WRITE_DRAFT, + // Rebasing writes a fresh draft, so drafts.rs `require_can_write_path` applies per kind. + requires: NONE, + kindRequires: { + script: WRITE_DRAFT, + flow: WRITE_FLOW_DRAFT, + app: WRITE_DRAFT, + resource: WRITE_DRAFT, + variable: WRITE_DRAFT, + schedule: WRITE_DRAFT, + trigger: WRITE_DRAFT + } satisfies Record<(typeof ITEM_TYPES)[number], SessionToolPolicy>, def: createToolDef( rebaseDraftSchema, 'rebase_draft', @@ -4218,13 +4299,14 @@ export const globalTools: SessionTool<{}>[] = [ kindRequires: { // scripts.rs `delete_script_by_path` calls `require_admin`; a non-admin archives. script: ['admin'], - // flows.rs `delete_flow_by_path` and apps.rs `delete_app` refuse operators. - flow: ['deploy', 'manage_code'], + // apps.rs `delete_app` refuses operators, flows.rs `delete_flow_by_path` those + // without builder rights. + flow: ['deploy', 'manage_flows'], app: ['deploy', 'manage_code'], resource: DEPLOY, variable: DEPLOY, - schedule: NONE, - trigger: NONE + schedule: ['manage_schedules'], + trigger: ['manage_triggers'] } satisfies Record<(typeof ITEM_TYPES)[number], SessionToolPolicy>, def: createToolDef( deleteWorkspaceItemSchema, @@ -4495,17 +4577,20 @@ export const globalTools: SessionTool<{}>[] = [ { requires: NONE, def: createToolDef(openPreviewSchema, 'open_preview', OPEN_PREVIEW_DESCRIPTION), - // Withhold the editor side from a session that cannot write drafts, so the model - // never offers the user a panel it would not get. + // Withhold the editors a session cannot save, so the model never offers the user a + // panel it would not get. schemaFor: async (helpers) => { const access = (helpers as GlobalToolHelpers | undefined)?.access - return access && !access.has('write_draft') - ? createToolDef( + if (canOpenEditor(access, 'script')) { + return createToolDef(openPreviewSchema, 'open_preview', OPEN_PREVIEW_DESCRIPTION) + } + return canOpenEditor(access, 'flow') + ? createToolDef(openPreviewFlowEditorSchema, 'open_preview', OPEN_PREVIEW_DESCRIPTION) + : createToolDef( openPreviewViewOnlySchema, 'open_preview', OPEN_PREVIEW_VIEW_ONLY_DESCRIPTION ) - : createToolDef(openPreviewSchema, 'open_preview', OPEN_PREVIEW_DESCRIPTION) }, fn: async (ctx) => { const parsed = openPreviewSchema.parse(ctx.args) @@ -4841,22 +4926,25 @@ async function openSessionPreview( if (!openPreviewHandler) { return 'Error: open_preview is only available inside an AI session. Tell the user to switch to a session to view the preview, or describe the item textually.' } - // A pipeline has only an editor, and its own capability gates already cover it. + // A pipeline has only an editor, so a session that cannot save its nodes has nothing to open. if (args.kind === 'pipeline') { + if (!canOpenEditor(access, 'pipeline')) { + return 'Error: you cannot open the pipeline editor in this workspace: this user cannot save drafts. Show its scripts with open_preview mode "view" instead.' + } return await openPreviewHandler({ ...args, mode: undefined, sessionId }) } // Resolved before the gates, not after: the handler reads a missing mode as the editor, so // gating on `args.mode === 'edit'` would wave through every call that simply omitted it — // which is what a model whose schema only offers 'view' does. const mode: 'edit' | 'view' = args.mode ?? 'edit' - // The advertised schema already withholds 'edit' from a session that cannot write - // drafts; re-check so a model asking outside the enum can't act outside it either. - if (mode === 'edit' && access && !access.has('write_draft')) { + // The advertised schema already narrows 'edit' to what this session can save; re-check so + // a model asking outside the enum can't act outside it either. + if (mode === 'edit' && !canOpenEditor(access, args.kind)) { const opened = await openPreviewHandler({ ...args, mode: 'view', sessionId }) - return `${opened}\nOpened the deployed page instead: you cannot open item editors in this workspace.` + return `${opened}\nOpened the deployed page instead: you cannot open ${args.kind === 'flow' ? 'flow' : 'item'} editors in this workspace.` } - // `write_draft` is a workspace capability and says nothing about this path, so an - // edit that survived it is still checked against the item's own permissions. + // The draft capability is workspace-wide and says nothing about this path, so an edit + // that survived it is still checked against the item's own permissions. if (mode === 'edit' && workspace) { const verdict = await canEditItemPath(workspace, args.kind, args.path) if (verdict !== 'allowed') { @@ -4884,10 +4972,12 @@ async function canEditItemPath( const role = await roleForWorkspace(workspace) if (role.kind === 'not_a_member') return 'denied' if (role.kind !== 'resolved' || !role.user) return 'unverified' - if (role.user.operator) return 'denied' + const user = role.user + const settings = get(userWorkspaces).find((w) => w.id === workspace)?.operator_settings + if (!roleCanDraft(kind, user, settings)) return 'denied' // Folder and ownership rules answer most calls without a request. An item's own // `extra_perms` can only widen them, so the item is fetched only to overturn a denial. - if (canWrite(path, {}, role.user)) return 'allowed' + if (canDraftItem(kind, path, {}, user, settings)) return 'allowed' try { const extraPerms = kind === 'script' @@ -4895,7 +4985,7 @@ async function canEditItemPath( : kind === 'flow' ? (await FlowService.getFlowByPath({ workspace, path })).extra_perms : (await AppService.getAppByPath({ workspace, path })).extra_perms - return canWrite(path, extraPerms ?? {}, role.user) ? 'allowed' : 'denied' + return canDraftItem(kind, path, extraPerms, user, settings) ? 'allowed' : 'denied' } catch (e) { // Nothing deployed means no sharing to widen the folder rules with. return (e as { status?: number } | null | undefined)?.status === 404 ? 'denied' : 'unverified' diff --git a/frontend/src/lib/components/copilot/chat/global/gate.ts b/frontend/src/lib/components/copilot/chat/global/gate.ts index d1dafe29ed..506f230027 100644 --- a/frontend/src/lib/components/copilot/chat/global/gate.ts +++ b/frontend/src/lib/components/copilot/chat/global/gate.ts @@ -25,14 +25,35 @@ export function isGlobalAiEnabled(): boolean { } } +const OPERATOR_SESSIONS_KEY = 'wm_operator_ai_sessions' + +/** + * Operators get AI sessions only in a browser that opted in, until the operator experience + * has been tested; everywhere else they keep the docked chat and `/sessions` refuses them. + * Set with `localStorage.setItem('wm_operator_ai_sessions', '1')` and reload. To ship it, + * delete this and `sessionsAllowedFor`, and drop the `isOperator` argument from every caller. + */ +function operatorSessionsEnabled(): boolean { + if (typeof localStorage === 'undefined') return false + try { + return localStorage.getItem(OPERATOR_SESSIONS_KEY) === '1' + } catch { + return false + } +} + +/** Whether this user's role may use AI sessions at all. */ +export function sessionsAllowedFor(isOperator: boolean | undefined): boolean { + return !isOperator || operatorSessionsEnabled() +} + /** * Whether an AI entry point hands off to a session instead of driving the docked * chat. Deliberately the same condition as the root layout's `disableAi`, so a * caller falling back on `false` always has a mounted pane to fall back to. - * Operators keep that pane (`/sessions` refuses them) until the operator chat ships. */ export function prefersSessionHandoff(isOperator: boolean | undefined): boolean { - return isGlobalAiEnabled() && !isOperator + return isGlobalAiEnabled() && sessionsAllowedFor(isOperator) } /** Clear the opt-out, then hard-reload so every gated site re-reads it. */ diff --git a/frontend/src/lib/components/copilot/chat/global/sessionAccess.test.ts b/frontend/src/lib/components/copilot/chat/global/sessionAccess.test.ts index 82ad8162d3..2e28375cee 100644 --- a/frontend/src/lib/components/copilot/chat/global/sessionAccess.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/sessionAccess.test.ts @@ -8,7 +8,7 @@ const { whoami, deployRules } = vi.hoisted(() => ({ vi.mock('$lib/gen', () => ({ UserService: { whoami } })) vi.mock('$lib/utils_workspace_deploy', () => ({ checkDeployRules: deployRules })) -import { resolveSessionAccess } from './sessionAccess' +import { capabilitiesForRole, resolveSessionAccess } from './sessionAccess' import { clearWorkspaceRoleCache } from '$lib/user' type WhoamiOverrides = { is_admin?: boolean; is_super_admin?: boolean; operator?: boolean } @@ -45,20 +45,30 @@ describe('resolveSessionAccess', () => { it('gives a developer every capability but admin', async () => { const caps = await capabilitiesFor({}) - expect([...caps].sort()).toEqual(['deploy', 'manage_code', 'run_preview', 'write_draft']) + expect([...caps].sort()).toEqual([ + 'deploy', + 'manage_code', + 'manage_flows', + 'manage_schedules', + 'manage_triggers', + 'run_flow_preview', + 'run_preview', + 'write_draft', + 'write_flow_draft' + ]) }) // The folder, resource and variable handlers run the rules but have no operator check, // so withholding `deploy` from an operator would be stricter than the server. it('leaves an operator the deploy-rule capability, and nothing their token refuses', async () => { const caps = await capabilitiesFor({ operator: true }) - expect([...caps]).toEqual(['deploy']) + expect([...caps].sort()).toEqual(['deploy', 'manage_schedules', 'manage_triggers']) }) it('takes deploy from the rules alone, so a rule blocks an operator too', async () => { deployRules.mockResolvedValue({ ok: false, refusedBy: 'DisableDirectDeployment' }) const caps = await capabilitiesFor({ operator: true }) - expect([...caps]).toEqual([]) + expect([...caps].sort()).toEqual(['manage_schedules', 'manage_triggers']) }) // Both spellings of `authed.is_admin`, which the draft path honours and the handlers @@ -67,7 +77,14 @@ describe('resolveSessionAccess', () => { 'lets an admin who is also an operator draft and deploy, but not preview or manage code (%o)', async (role) => { const caps = await capabilitiesFor({ ...role, operator: true }) - expect([...caps].sort()).toEqual(['admin', 'deploy', 'write_draft']) + expect([...caps].sort()).toEqual([ + 'admin', + 'deploy', + 'manage_schedules', + 'manage_triggers', + 'write_draft', + 'write_flow_draft' + ]) } ) @@ -82,6 +99,37 @@ describe('resolveSessionAccess', () => { expect(caps.has('write_draft')).toBe(true) }) + // flows.rs, drafts.rs and run_preview_flow all let a builder through for flows, and + // nothing else moves: scripts, apps and their previews stay refused. + it("gives a builder operator the flow capabilities and nothing a script's handler refuses", () => { + const caps = capabilitiesForRole({ + isAdmin: false, + operator: true, + deployRulesPass: true, + operatorSettings: { builder_flows: true } + }) + expect([...caps].sort()).toEqual([ + 'deploy', + 'manage_flows', + 'manage_schedules', + 'manage_triggers', + 'run_flow_preview', + 'write_flow_draft' + ]) + }) + + // A withdrawable right: only `false` withdraws it, as `gate_operator_writes` reads it. + it('withholds schedule and trigger writes only where the workspace set them to false', () => { + const caps = capabilitiesForRole({ + isAdmin: false, + operator: true, + deployRulesPass: true, + operatorSettings: { manage_schedules: false } + }) + expect(caps.has('manage_schedules')).toBe(false) + expect(caps.has('manage_triggers')).toBe(true) + }) + it('grants everything when the role cannot be resolved', async () => { whoami.mockRejectedValueOnce(new Error('network')) const access = await resolveSessionAccess('ws') diff --git a/frontend/src/lib/components/copilot/chat/global/sessionAccess.ts b/frontend/src/lib/components/copilot/chat/global/sessionAccess.ts index 1ac13136a4..2f1e00c09b 100644 --- a/frontend/src/lib/components/copilot/chat/global/sessionAccess.ts +++ b/frontend/src/lib/components/copilot/chat/global/sessionAccess.ts @@ -1,12 +1,26 @@ +import { get } from 'svelte/store' +import { userWorkspaces } from '$lib/stores' +import type { OperatorSettings } from '$lib/gen' import { getWorkspaceRole } from '$lib/user' +import { roleCanAuthor, roleCanDraft } from '$lib/editRights' import { checkDeployRules } from '$lib/utils_workspace_deploy' import type { SessionAccess, SessionCapability } from '../sessionCapabilities' +type OperatorRights = Pick< + NonNullable, + 'builder_flows' | 'manage_schedules' | 'manage_triggers' +> + const ALL_CAPABILITIES: SessionCapability[] = [ 'write_draft', + 'write_flow_draft', 'run_preview', + 'run_flow_preview', 'deploy', 'manage_code', + 'manage_flows', + 'manage_schedules', + 'manage_triggers', 'admin' ] @@ -25,19 +39,35 @@ export function capabilitiesForRole(role: { isAdmin: boolean operator: boolean deployRulesPass: boolean + /** The workspace's `operator_settings`, null for a non-operator. */ + operatorSettings?: OperatorRights | null }): SessionAccess { const capabilities = new Set() - // Per-capability precedence, NOT a role ladder: drafts.rs `require_can_write_path` - // returns Ok on `authed.is_admin` BEFORE its operator branch, while jobs.rs - // `run_preview_*` and the script/flow/app handlers refuse `authed.is_operator` with no - // admin escape — and on the session path that flag is never cleared for an admin. - if (role.isAdmin || !role.operator) { + const user = { operator: role.operator, is_admin: role.isAdmin, is_super_admin: false } + const authorsCode = roleCanAuthor('script', user, role.operatorSettings) + const authorsFlows = roleCanAuthor('flow', user, role.operatorSettings) + // Drafts take the admin precedence `roleCanDraft` encodes; previews and the item handlers + // do not — and on the session path the operator flag is never cleared for an admin. + if (roleCanDraft('script', user, role.operatorSettings)) { capabilities.add('write_draft') } - if (!role.operator) { + if (roleCanDraft('flow', user, role.operatorSettings)) { + capabilities.add('write_flow_draft') + } + if (authorsCode) { capabilities.add('run_preview') capabilities.add('manage_code') } + if (authorsFlows) { + capabilities.add('run_flow_preview') + capabilities.add('manage_flows') + } + if (roleCanAuthor('schedule', user, role.operatorSettings)) { + capabilities.add('manage_schedules') + } + if (roleCanAuthor('trigger', user, role.operatorSettings)) { + capabilities.add('manage_triggers') + } // No operator term: the rules are their own gate, and the handlers that also refuse // operators say so through `manage_code`. Admins bypass the rules inside the check. if (role.deployRulesPass) { @@ -63,6 +93,7 @@ export async function resolveSessionAccess(workspace: string): Promise w.id === workspace)?.operator_settings }) } diff --git a/frontend/src/lib/components/copilot/chat/global/sessionToolset.test.ts b/frontend/src/lib/components/copilot/chat/global/sessionToolset.test.ts index 143def2fc8..0c0b8d8a7d 100644 --- a/frontend/src/lib/components/copilot/chat/global/sessionToolset.test.ts +++ b/frontend/src/lib/components/copilot/chat/global/sessionToolset.test.ts @@ -75,10 +75,22 @@ const REACHABLE_PROFILES = [ [false, true].flatMap((isAdmin) => [false, true].flatMap((operator) => // `checkDeployRules` bypasses on admin, so an admin has only the one outcome. - (isAdmin ? [true] : [false, true]).map((deployRulesPass) => { - const access = capabilitiesForRole({ isAdmin, operator, deployRulesPass }) - return [[...access].sort().join(',') || 'none', access] as const - }) + (isAdmin ? [true] : [false, true]).flatMap((deployRulesPass) => + // The operator rights a workspace can set: none, the builder right, both locks. + [ + undefined, + { builder_flows: true }, + { manage_schedules: false, manage_triggers: false } + ].map((operatorSettings) => { + const access = capabilitiesForRole({ + isAdmin, + operator, + deployRulesPass, + operatorSettings + }) + return [[...access].sort().join(',') || 'none', access] as const + }) + ) ) ) ).entries() @@ -122,13 +134,24 @@ describe('session tool policies', () => { const kindsOf = (name: string, access: SessionAccess) => (shipped(name, access).def.function.parameters as any).properties.type.enum + const developerCaps: SessionCapability[] = [ + 'write_draft', + 'write_flow_draft', + 'run_preview', + 'run_flow_preview', + 'manage_code', + 'manage_flows', + 'manage_schedules', + 'manage_triggers' + ] + // A developer a protection rule refuses keeps the two kinds no rule reaches. - const refused = accessWith(['write_draft', 'run_preview', 'manage_code']) + const refused = accessWith(developerCaps) expect(kindsOf('deploy_workspace_item', refused)).toEqual(['schedule', 'trigger']) expect(kindsOf('delete_workspace_item', refused)).toEqual(['schedule', 'trigger']) // An operator where no rule applies: only the code handlers refuse them. - const operator = accessWith(['deploy']) + const operator = accessWith(['deploy', 'manage_schedules', 'manage_triggers']) expect(kindsOf('deploy_workspace_item', operator)).toEqual([ 'schedule', 'trigger', @@ -136,8 +159,17 @@ describe('session tool policies', () => { 'variable' ]) + // A workspace that withdrew schedule writes from operators: `gate_operator_writes`. + const locked = accessWith(['deploy', 'manage_triggers']) + expect(kindsOf('delete_workspace_item', locked)).toEqual(['trigger', 'resource', 'variable']) + + // A builder operator deploys flows too, and still no script or app. + const builder = accessWith(['deploy', 'manage_flows', 'manage_schedules', 'manage_triggers']) + expect(kindsOf('deploy_workspace_item', builder)).toContain('flow') + expect(kindsOf('deploy_workspace_item', builder)).not.toContain('script') + // Deleting a script is admin-only, where deploying one is not. - const developer = accessWith(['write_draft', 'run_preview', 'manage_code', 'deploy']) + const developer = accessWith([...developerCaps, 'deploy']) expect(kindsOf('deploy_workspace_item', developer)).toContain('script') expect(kindsOf('delete_workspace_item', developer)).not.toContain('script') diff --git a/frontend/src/lib/components/copilot/chat/sessionCapabilities.ts b/frontend/src/lib/components/copilot/chat/sessionCapabilities.ts index fc77b3e1c4..f96e89dddc 100644 --- a/frontend/src/lib/components/copilot/chat/sessionCapabilities.ts +++ b/frontend/src/lib/components/copilot/chat/sessionCapabilities.ts @@ -12,14 +12,27 @@ export type SessionCapability = /** Passes drafts.rs `require_can_write_path` — for a draft of any kind, schedules and * triggers included, though an operator may create those directly. */ | 'write_draft' + /** Passes `require_can_write_path` for a flow draft, which an operator also passes where the + * workspace grants `builder_flows`. The server then refuses a draft carrying inline code. */ + | 'write_flow_draft' /** Passes the operator refusal jobs.rs `run_preview_*` makes before starting a job. */ | 'run_preview' + /** Passes `run_preview_flow`'s `check_operator_can_build_flows`: composed flows only, for an + * operator with `builder_flows`. */ + | 'run_flow_preview' /** Passes `check_deploy_rules`. */ | 'deploy' - /** Passes the operator refusal the script, flow and app handlers make before creating or + /** Passes the operator refusal the script and app handlers make before creating or * deleting one. The same condition as `run_preview` today, but another handler family's * check, so each follows its own gate if the two ever diverge. */ | 'manage_code' + /** Passes the flow handlers' `check_operator_can_build_flows`. */ + | 'manage_flows' + /** Passes `gate_operator_writes` on the schedule router: withheld only from an operator whose + * workspace set `operator_settings.manage_schedules` to false. */ + | 'manage_schedules' + /** The same gate on the trigger routers, read from `manage_triggers`. */ + | 'manage_triggers' /** Passes `require_admin`. */ | 'admin' @@ -36,9 +49,22 @@ export type SessionToolPolicy = readonly SessionCapability[] export const NONE: SessionToolPolicy = [] export const WRITE_DRAFT: SessionToolPolicy = ['write_draft'] +export const WRITE_FLOW_DRAFT: SessionToolPolicy = ['write_flow_draft'] export const RUN_PREVIEW: SessionToolPolicy = ['run_preview'] +export const RUN_FLOW_PREVIEW: SessionToolPolicy = ['run_flow_preview'] export const DEPLOY: SessionToolPolicy = ['deploy'] +/** Editors a session can open. Every editor autosaves a draft, so it needs the draft + * capability of its kind; a pipeline's nodes are script drafts. */ +export type EditorKind = 'script' | 'flow' | 'raw_app' | 'pipeline' + +/** Whether `access` can save what the `kind` editor writes. `undefined` access (not + * resolved yet, or not a session) answers true, as the toolset filter does. */ +export function canOpenEditor(access: SessionAccess | undefined, kind: EditorKind): boolean { + if (!access) return true + return access.has(kind === 'flow' ? 'write_flow_draft' : 'write_draft') +} + /** * A tool that can reach an AI session's toolset. `requires` is mandatory, and declared * beside the `fn` whose call is what proves it right. diff --git a/frontend/src/lib/components/copilot/chat/skills/skillResources.ts b/frontend/src/lib/components/copilot/chat/skills/skillResources.ts index 7a80705d4e..d28a4d32e8 100644 --- a/frontend/src/lib/components/copilot/chat/skills/skillResources.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skillResources.ts @@ -1,6 +1,7 @@ import { ResourceService } from '$lib/gen' -import { canWrite } from '$lib/utils' -import type { UserExt } from '$lib/stores' +import { canEditItem } from '$lib/editRights' +import { userWorkspaces, type UserExt } from '$lib/stores' +import { get } from 'svelte/store' /** * Skills are resources of this type: a file resource (`format_extension = 'md'`) @@ -53,9 +54,10 @@ export type SkillListing = { skills: SkillResource[]; truncated: boolean } * `user` decides which rows the drawer offers to edit rather than only view; pass * the account the workspace is being browsed as. Ownership is mostly implicit in * the path (`u//…`, a folder the user owns), which is why this goes through - * the shared `canWrite` rather than reading `extra_perms` alone. */ + * the shared `canEditItem` rather than reading `extra_perms` alone. */ export async function listSkillResources(workspace: string, user?: UserExt): Promise { if (!workspace) return { skills: [], truncated: false } + const operatorSettings = get(userWorkspaces).find((w) => w.id === workspace)?.operator_settings const rows: SkillResource[] = [] for (let page = 1; page <= MAX_SKILLS_PAGES; page++) { const resources = await ResourceService.listResource({ @@ -70,7 +72,7 @@ export async function listSkillResources(workspace: string, user?: UserExt): Pro name: skillNameFromPath(r.path), description: r.description ?? '', editedAt: r.edited_at, - canWrite: canWrite(r.path, r.extra_perms ?? {}, user) + canWrite: canEditItem('resource', r.path, r.extra_perms, user, operatorSettings) })) ) if (resources.length < SKILLS_PAGE_SIZE) return { skills: rows, truncated: false } diff --git a/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts b/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts index dd480c5e97..75d996ebef 100644 --- a/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts +++ b/frontend/src/lib/components/copilot/chat/skills/skillsMenu.svelte.ts @@ -1,4 +1,4 @@ -import { BookOpen, List, Plus } from 'lucide-svelte' +import { BookOpen, List, Loader2, Plus } from 'lucide-svelte' import { get } from 'svelte/store' import { userStore, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' @@ -14,6 +14,14 @@ type Row = SkillResource & { enabled: boolean } // scannable, so the rest are reached through the settings modal rather than dropped. const MAX_MENU_SKILLS = 8 +// Not `disabled`: melt stamps that on a row at mount and never clears it, and the +// unkeyed submenu reuses this row for the first skill once the load lands. +const loadingItem: Item = { + displayName: 'Loading…', + icon: Loader2, + iconProps: { class: 'animate-spin shrink-0' } +} + /** * The chat "+" menu's Skills submenu: one row per skill, checked when it is on, * then the way to manage them. Everything a skill is beyond turning it on and off @@ -25,7 +33,7 @@ export class SkillsMenu { #seq = 0 /** Rows for the workspace named by `#rowsWorkspace`, and meaningless for any other. */ #rows = $state([]) - #rowsWorkspace: string | undefined = undefined + #rowsWorkspace = $state(undefined) constructor(manager: AIChatManager, onManage: () => void) { this.#manager = manager @@ -89,19 +97,21 @@ export class SkillsMenu { await this.#manager.refreshGlobalSkills(ws) } - /** Loaded on open so the checks are current. */ - async items(closeMenu?: () => void): Promise { + /** Called on open so the checks are current; never awaited, see `items`. */ + refresh() { + const ws = this.#ws + if (ws) void this.#load(ws) + } + + /** + * Built from the live rows, so call it from a getter: the submenu then fills in + * when a load lands instead of the whole "+" menu waiting on the fetch. Rows for + * another workspace are not shown — same path, different skill. + */ + items(closeMenu?: () => void): Item[] { const ws = this.#ws if (!ws) return [] - // The menu opens on what is already known and refreshes behind it: awaited - // inline it would stall the whole "+" menu, attachments included. Rows for - // another workspace are not "already known" — same path, different skill. - if (this.#rowsWorkspace !== ws) { - this.#rows = [] - await this.#load(ws) - } else { - void this.#load(ws) - } + if (this.#rowsWorkspace !== ws) return [loadingItem] const ambiguous = ambiguousSkillNames(this.#rows) // By path. Ordering the ones that are on first would put every row in the same // bucket now that skills start that way, and drop the one row it did move — a diff --git a/frontend/src/lib/components/dbTableFilters.ts b/frontend/src/lib/components/dbTableFilters.ts index 9d9c83e6b0..c3266b3ac7 100644 --- a/frontend/src/lib/components/dbTableFilters.ts +++ b/frontend/src/lib/components/dbTableFilters.ts @@ -52,7 +52,7 @@ export function buildDbTableFilterSchema(columns: ColumnDef[]): DbTableFilterSch const columnOfKey: Record = {} const keyOfColumn: Record = {} for (const col of columns) { - if (!col.field) continue + if (!col.field || col.filterable === false) continue let base = col.field.replace(/\W+/g, '_') || 'column' if (base === FREE_TEXT_KEY) base = `${base}col` let key = base diff --git a/frontend/src/lib/components/dbt/DbtModelGraph.svelte b/frontend/src/lib/components/dbt/DbtModelGraph.svelte index bd7f42a9e7..12b008812d 100644 --- a/frontend/src/lib/components/dbt/DbtModelGraph.svelte +++ b/frontend/src/lib/components/dbt/DbtModelGraph.svelte @@ -16,7 +16,6 @@ import { Button } from '$lib/components/common' import { Loader2, RefreshCw } from 'lucide-svelte' import { displayDate } from '$lib/utils' - import { base } from '$lib/base' import AssetGraphCanvas from '$lib/components/assets/AssetGraph/AssetGraphCanvas.svelte' import type { DbtPreviewBuffer } from './previewRows' import type { @@ -477,7 +476,7 @@ worker tag nothing serves waits here indefinitely. Open the parse job @@ -491,7 +490,7 @@ {#if refreshError.job} Open the parse job diff --git a/frontend/src/lib/components/details/FlowDetail.svelte b/frontend/src/lib/components/details/FlowDetail.svelte index ce7378ede4..30de5dc7cb 100644 --- a/frontend/src/lib/components/details/FlowDetail.svelte +++ b/frontend/src/lib/components/details/FlowDetail.svelte @@ -23,13 +23,12 @@ import DetailPageLayout from '$lib/components/details/DetailPageLayout.svelte' import OnBehalfOfBadge from '$lib/components/details/OnBehalfOfBadge.svelte' import { goto } from '$lib/navigation' - import { base } from '$lib/base' import { Badge as HeaderBadge, Alert } from '$lib/components/common' import MoveDrawer from '$lib/components/MoveDrawer.svelte' import RunForm from '$lib/components/RunForm.svelte' import ShareModal from '$lib/components/ShareModal.svelte' import { enterpriseLicense, userStore, userWorkspaces, workspaceStore } from '$lib/stores' - import { useOperatorBuilderFlows } from '$lib/operatorWriteRights' + import { useEditRights } from '$lib/operatorWriteRights' import { sendUserToast } from '$lib/toast' import DeployWorkspaceDrawer from '$lib/components/DeployWorkspaceDrawer.svelte' import SavedInputsV2 from '$lib/components/SavedInputsV2.svelte' @@ -131,7 +130,7 @@ // A ⌘-click opens these in a new tab, which only the query names the workspace for. const operatingHref = useOperatingWorkspaceHref() const actingUser = $derived(operatingUser.current) - const operatorBuilderFlows = useOperatorBuilderFlows() + const editRights = useEditRights() let flow: Flow | undefined = $state() // Derived, not assigned during the load: in a fork the acting user is looked up @@ -345,9 +344,7 @@ } } - // Operators with the builder right author flows out of deployed runnables; every other - // operator is read-only here. - let canAuthorFlow = $derived(!actingUser?.operator || $operatorBuilderFlows) + let canAuthorFlow = $derived(editRights.roleCanAuthor('flow')) let moveDrawer: MoveDrawer | undefined = $state() let deploymentDrawer: DeployWorkspaceDrawer | undefined = $state() @@ -374,7 +371,7 @@ description: `Start a new flow from a copy of this one`, narrow: { dropdownOf: 'Edit' }, buttonProps: { - href: operatingHref(`${base}/flows/add?template=${flow.path}`), + href: operatingHref(`/flows/add?template=${flow.path}`), onClick: interceptNav(onNavigate, `/flows/add?template=${flow.path}`), variant: 'subtle', unifiedSize: 'md', @@ -411,7 +408,7 @@ buttons.push({ label: `Runs`, buttonProps: { - href: operatingHref(`${base}/runs/${flow.path}`), + href: operatingHref(`/runs/${flow.path}`), onClick: interceptNav(onNavigate, `/runs/${flow.path}`), unifiedSize: 'md', variant: 'subtle', @@ -434,7 +431,7 @@ return buttons } - // The builder right covers flows only; building an app is still refused to operators. + // Build app makes a low-code app, which no builder right covers. if (!actingUser?.operator) { buttons.push({ label: 'Build app', @@ -457,7 +454,7 @@ buttons.push({ label: 'Edit', buttonProps: { - href: operatingHref(`${base}/flows/edit/${path}`), + href: operatingHref(`/flows/edit/${path}`), onClick: interceptNav(onNavigate, `/flows/edit/${path}`), variant: 'accent', unifiedSize: 'md', diff --git a/frontend/src/lib/components/details/ScriptDetail.svelte b/frontend/src/lib/components/details/ScriptDetail.svelte index f33f00fb8f..6712092c39 100644 --- a/frontend/src/lib/components/details/ScriptDetail.svelte +++ b/frontend/src/lib/components/details/ScriptDetail.svelte @@ -1,5 +1,4 @@ diff --git a/frontend/src/lib/components/flows/CreateActionsFlow.svelte b/frontend/src/lib/components/flows/CreateActionsFlow.svelte index ae848156f5..9f78585cc0 100644 --- a/frontend/src/lib/components/flows/CreateActionsFlow.svelte +++ b/frontend/src/lib/components/flows/CreateActionsFlow.svelte @@ -1,6 +1,5 @@ diff --git a/frontend/src/lib/components/flows/agentDraft.svelte.ts b/frontend/src/lib/components/flows/agentDraft.svelte.ts index b76f9bb471..cb6081ef46 100644 --- a/frontend/src/lib/components/flows/agentDraft.svelte.ts +++ b/frontend/src/lib/components/flows/agentDraft.svelte.ts @@ -3,8 +3,8 @@ import { get } from 'svelte/store' import { deepEqual } from 'fast-equals' import { ResourceService } from '$lib/gen' import { sendUserToast } from '$lib/toast' -import { canWrite } from '$lib/utils' -import { userStore } from '$lib/stores' +import { canEditItem, roleCanAuthor } from '$lib/editRights' +import { userStore, userWorkspaces } from '$lib/stores' import { getUserExt } from '$lib/user' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' import { UserDraft } from '$lib/userDraft.svelte' @@ -339,10 +339,12 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { // Same rule the generic resource editor applies. The backend refuses the write // either way, but without this the editor would invite edits it cannot save and // autosave a draft on every keystroke against a resource the reader cannot deploy. - canWriteResource = canWrite( + canWriteResource = canEditItem( + 'agent', r.path, - r.extra_perms ?? {}, - user ?? get(userStore) ?? undefined + r.extra_perms, + user ?? get(userStore) ?? undefined, + get(userWorkspaces).find((w) => w.id === ws)?.operator_settings ) // An agent that exists only as a draft has no deployed value to compare against or // fall back to, and the response's is a synthetic echo of the draft. Leaving the @@ -382,7 +384,11 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { if (loadedFor !== key) return noDeployed = true deployed = undefined - canWriteResource = true + canWriteResource = roleCanAuthor( + 'agent', + get(userStore) ?? undefined, + get(userWorkspaces).find((w) => w.id === ws)?.operator_settings + ) state = { path: name, description: '', @@ -438,6 +444,9 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { // The counter the step card's write-back used to report, from the surface that now owns the // write: a deploy here reaches every flow linking this agent. logReusableAgentUsage(noDeployed ? 'saved' : 'updated') + // A new agent's editor opens on its draft's storage path, so its first save always lands + // elsewhere: that is naming it, not a rename. + const firstSave = noDeployed deployed = submitted noDeployed = false const renamed = submitted.path !== fromPath @@ -456,7 +465,9 @@ export function useAgentDraft(opts: AgentDraftOptions): AgentDraftHandle { sync.discard(fromPath, submitted) // A rename moves the row, so the next load must not reuse the old key. loadedFor = `${ws}:${submitted.path}` - sendUserToast(renamed ? `Renamed agent to ${submitted.path}` : `Saved agent ${submitted.path}`) + sendUserToast( + renamed && !firstSave ? `Renamed agent to ${submitted.path}` : `Saved agent ${submitted.path}` + ) return submitted.path } diff --git a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte index 9c2a57b43f..658a7adb77 100644 --- a/frontend/src/lib/components/flows/content/AgentEditorHost.svelte +++ b/frontend/src/lib/components/flows/content/AgentEditorHost.svelte @@ -9,7 +9,8 @@ AGENT_CHAT_SCHEMA, agentChatFlow, agentChatGap, - agentChatPath + agentChatPath, + AGENT_STEP_ID } from '../conversations/agentEditorChat' import { runFlowPreview } from '../utils.svelte' import { goto } from '$lib/navigation' @@ -100,7 +101,7 @@ * schema per tool — the linked branch deliberately loads none. Reserved in `forbiddenIds`: * the root shares a flow-state map with the tools, so a tool of this id would lose its * schema to the root's. */ - const AGENT_ID = '__wm_agent_root' + const AGENT_ID = AGENT_STEP_ID const draft = useAgentDraft({ path: () => path, diff --git a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte index dc7c0948eb..3359f20080 100644 --- a/frontend/src/lib/components/flows/content/AgentResourceBar.svelte +++ b/frontend/src/lib/components/flows/content/AgentResourceBar.svelte @@ -8,8 +8,8 @@ import ResourcePathHint from '$lib/components/ResourcePathHint.svelte' import { ResourceService, type InputTransform, type Resource } from '$lib/gen' import { sendUserToast } from '$lib/toast' - import { userStore } from '$lib/stores' - import { canWrite } from '$lib/utils' + import { userStore, userWorkspaces } from '$lib/stores' + import { canEditItem } from '$lib/editRights' import { Bot, ChevronDown, ChevronUp, Save, Unlink, Pencil } from 'lucide-svelte' import { AGENT_BRAIN_KEYS, @@ -235,8 +235,13 @@ let unavailable = $derived(linkedInfo?.unavailable ?? false) let canEditAgent = $derived( !!agent && - !$userStore?.operator && - canWrite(agent, linkedInfo?.extraPerms ?? {}, $userStore ?? undefined) + canEditItem( + 'agent', + agent, + linkedInfo?.extraPerms, + $userStore ?? undefined, + $userWorkspaces.find((w) => w.id === ws)?.operator_settings + ) ) // The hint flips on the first keystroke in the agent editor, so the badge does not wait for the // debounced autosave and the refetch behind it; the fetched answer covers a draft written diff --git a/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte b/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte index 979717dae1..9002e88adf 100644 --- a/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte +++ b/frontend/src/lib/components/flows/content/ExpandedSubflowStep.svelte @@ -13,7 +13,6 @@ type ResolvedExpandedSubflowStep } from '../expandedSubflowStep' import { parseExpandedSubflowId } from '$lib/components/restartFromStepPath' - import { base } from '$app/paths' import FlowPanelChrome from '../common/FlowPanelChrome.svelte' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' @@ -122,7 +121,7 @@ title="Open the subflow in a new tab" startIcon={{ icon: ExternalLink }} iconOnly - href={`${base}/flows/edit/${containingFlowPath}?workspace=${encodeURIComponent( + href={`/flows/edit/${containingFlowPath}?workspace=${encodeURIComponent( opWs ?? '' )}${module ? `&selected=${encodeURIComponent(leafId)}` : ''}`} target="_blank" diff --git a/frontend/src/lib/components/flows/content/FlowEditorDrawer.svelte b/frontend/src/lib/components/flows/content/FlowEditorDrawer.svelte index 6b4d73c6a0..c6c1d13b2a 100644 --- a/frontend/src/lib/components/flows/content/FlowEditorDrawer.svelte +++ b/frontend/src/lib/components/flows/content/FlowEditorDrawer.svelte @@ -11,7 +11,6 @@ import { initFlow } from '$lib/components/flows/flowStore.svelte' import type { FlowState } from '$lib/components/flows/flowState' import type { FlowEditorContext } from '../types' - import { base } from '$app/paths' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' const operatingWorkspace = useOperatingWorkspace() @@ -127,7 +126,7 @@ // Follow the drawer's own selection, which the user may have moved since it opened. const stepId = flowBuilder?.getSelectedId() ?? selectedStepId const selected = stepId ? `?selected=${encodeURIComponent(stepId)}` : '' - window.open(`${base}/flows/edit/${flowPath}${selected}`, '_blank', 'noopener,noreferrer') + window.open(`/flows/edit/${flowPath}${selected}`, '_blank', 'noopener,noreferrer') flowEditorDrawer?.closeDrawer() }} startIcon={{ icon: ExternalLink }} diff --git a/frontend/src/lib/components/flows/conversations/agentEditorChat.test.ts b/frontend/src/lib/components/flows/conversations/agentEditorChat.test.ts index c00c1e4297..99db5b8a9f 100644 --- a/frontend/src/lib/components/flows/conversations/agentEditorChat.test.ts +++ b/frontend/src/lib/components/flows/conversations/agentEditorChat.test.ts @@ -1,6 +1,6 @@ import { describe, expect, it } from 'vitest' import type { FlowModule } from '$lib/gen' -import { agentChatFlow, agentChatGap, agentChatPath } from './agentEditorChat' +import { agentChatFlow, agentChatGap, agentChatPath, agentStepLabel } from './agentEditorChat' describe('agentChatPath', () => { it('is a path no flow can take, so a same-path flow never shares its conversations', () => { @@ -10,6 +10,14 @@ describe('agentChatPath', () => { }) }) +describe('agentStepLabel', () => { + it("names a run's agent step by the agent, a chat turn's included", () => { + expect(agentStepLabel('f/support/agent')).toBe('f/support/agent') + expect(agentStepLabel(agentChatPath('f/support/agent'))).toBe('f/support/agent') + expect(agentStepLabel(undefined)).toBe('AI Agent') + }) +}) + function agent(inputTransforms: Record): FlowModule { return { id: '__wm_agent_root', diff --git a/frontend/src/lib/components/flows/conversations/agentEditorChat.ts b/frontend/src/lib/components/flows/conversations/agentEditorChat.ts index b754e04bb8..0f1bc86bda 100644 --- a/frontend/src/lib/components/flows/conversations/agentEditorChat.ts +++ b/frontend/src/lib/components/flows/conversations/agentEditorChat.ts @@ -13,6 +13,15 @@ export function agentChatPath(agentPath: string): string { return `${agentPath}.chat` } +/** The id of the one step a saved agent's run is made of, as the server names it too: job-read + * scoping and the run's agent detection match on it, so it is only ever relabeled for display. */ +export const AGENT_STEP_ID = '__wm_agent_root' + +/** What a run shows for that step: the agent it ran, read off the run's path. */ +export function agentStepLabel(runPath: string | undefined): string { + return runPath?.replace(/\.chat$/, '') || 'AI Agent' +} + /** * The inputs of the flow a saved agent is chatted with: the message, and the files the composer's * paperclip uploads. `user_message` is the name the server requires of a chat-mode run. diff --git a/frontend/src/lib/components/flows/conversations/messageAttachments.ts b/frontend/src/lib/components/flows/conversations/messageAttachments.ts index 36c7d103c3..18170a90df 100644 --- a/frontend/src/lib/components/flows/conversations/messageAttachments.ts +++ b/frontend/src/lib/components/flows/conversations/messageAttachments.ts @@ -1,6 +1,5 @@ /** The files a user message carried, as the lanes its bubble renders: image thumbnails and file chips. */ import type { ChatAttachment } from 'windmill-chat' -import { base } from '$lib/base' import { createAttachedFileContextElement, type ContextElement @@ -24,7 +23,7 @@ function looksLikeImage(attachment: ChatAttachment): boolean { function downloadUrl(workspace: string, attachment: ChatAttachment): string { const params = new URLSearchParams({ file_key: attachment.s3 }) if (attachment.storage) params.set('storage', attachment.storage) - return `${base}/api/w/${workspace}/job_helpers/download_s3_file?${params.toString()}` + return `/api/w/${workspace}/job_helpers/download_s3_file?${params.toString()}` } /** Without a workspace there is no link to build, so an image falls back to a file chip. */ diff --git a/frontend/src/lib/components/flows/linkedAgentDrafts.ts b/frontend/src/lib/components/flows/linkedAgentDrafts.ts index 4a9aef24b1..9367b21b12 100644 --- a/frontend/src/lib/components/flows/linkedAgentDrafts.ts +++ b/frontend/src/lib/components/flows/linkedAgentDrafts.ts @@ -3,11 +3,12 @@ import { type FlowModule, type FlowValue, type InputTransform, + type OperatorSettings, type Resource } from '$lib/gen' import { UserDraft } from '$lib/userDraft.svelte' import { UserDraftDbSyncer } from '$lib/userDraftDbSyncer.svelte' -import { canWrite } from '$lib/utils' +import { canEditItem } from '$lib/editRights' import type { UserExt } from '$lib/stores' import { dfs } from './dfs' import { flowLocalInputs, type AIAgentConfig } from './agentResourceUtils' @@ -122,11 +123,15 @@ export interface LinkedAgentDraft { extraPerms: Record } -/** Whether `user` may write this agent's resource. Split from the load so that resolving the - * drafts of a whole flow costs no `whoami` — only the deploy dialog needs the answer, and it - * looks the user up once for every agent it lists. */ -export function agentDraftCanWrite(draft: LinkedAgentDraft, user: UserExt | undefined): boolean { - return canWrite(draft.path, draft.extraPerms, user) +/** Whether `user` may write this agent's resource, in a workspace with these `operator_settings`. + * Split from the load so that resolving the drafts of a whole flow costs no `whoami` — only the + * deploy dialog needs the answer, and it looks the user up once for every agent it lists. */ +export function agentDraftCanWrite( + draft: LinkedAgentDraft, + user: UserExt | undefined, + operatorSettings: OperatorSettings | undefined +): boolean { + return canEditItem('agent', draft.path, draft.extraPerms, user, operatorSettings) } /** A link that cannot resolve for the user rather than because something went wrong: the agent was diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItemViewer.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItemViewer.svelte index a0ec4759f5..c87e20139e 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItemViewer.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItemViewer.svelte @@ -4,6 +4,7 @@ import type { FlowNodeColorClasses } from '$lib/components/graph' import { twMerge } from 'tailwind-merge' import { getContext } from 'svelte' + import { AGENT_STEP_ID } from '../conversations/agentEditorChat' // Detached mode (sessions): a step's editor is hidden until asked for, so add the // hint to the name tooltip. Provided by FlowEditor; absent (=> false) elsewhere. @@ -69,7 +70,8 @@
- {#if id && id !== 'preprocessor' && !id.startsWith('failure') && !id.startsWith('subflow:')} + + {#if id && id !== 'preprocessor' && id !== AGENT_STEP_ID && !id.startsWith('failure') && !id.startsWith('subflow:')} import type { Job } from '$lib/gen' import { twMerge } from 'tailwind-merge' - import { base } from '$lib/base' import { ExternalLink } from 'lucide-svelte' import { emptyString } from '$lib/utils' @@ -42,7 +41,7 @@ {#if !emptyString(job.id)} diff --git a/frontend/src/lib/components/flows/propPicker/OutputPickerInner.svelte b/frontend/src/lib/components/flows/propPicker/OutputPickerInner.svelte index aa9b14b762..e63db4e960 100644 --- a/frontend/src/lib/components/flows/propPicker/OutputPickerInner.svelte +++ b/frontend/src/lib/components/flows/propPicker/OutputPickerInner.svelte @@ -28,7 +28,6 @@ import { twMerge } from 'tailwind-merge' import DisplayResultControlBar from '$lib/components/DisplayResultControlBar.svelte' import LogViewer from '$lib/components/LogViewer.svelte' - import { base } from '$lib/base' import { fade } from 'svelte/transition' import type { FlowEditorContext, OutputViewerJob } from '../types' import { NEVER_TESTED_THIS_FAR } from '../models' @@ -564,7 +563,6 @@ { @@ -576,7 +574,6 @@ {:else if mock?.enabled && displayResultMock && toolbarLocationMock === 'external'} { diff --git a/frontend/src/lib/components/git_sync/GitPushPreview.svelte b/frontend/src/lib/components/git_sync/GitPushPreview.svelte index bed6d741d1..25207b767c 100644 --- a/frontend/src/lib/components/git_sync/GitPushPreview.svelte +++ b/frontend/src/lib/components/git_sync/GitPushPreview.svelte @@ -14,6 +14,7 @@ buttons — the setup dialog puts them in its own footer, the standalone modal i import hubPaths from '$lib/hubPaths.json' import { jobManager } from '$lib/services/JobManager' import type { SyncResponse, SettingsObject } from '$lib/git-sync' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' interface Props { gitRepoResourcePath: string @@ -39,7 +40,7 @@ buttons — the setup dialog puts them in its own footer, the standalone modal i let showCliInstructions = $state(false) const cliInstructions = $derived(`# Setup (only needed if local folder not initialized yet) npm install -g windmill-cli -wmill workspace add ${$workspaceStore} ${$workspaceStore} ${window.location.origin} +wmill workspace add ${$workspaceStore} ${$workspaceStore} ${apiBaseUrl()}/ wmill init --workspace ${$workspaceStore} --repository ${gitRepoResourcePath} # Pull workspace content to git repository diff --git a/frontend/src/lib/components/git_sync/GitSyncSection.svelte b/frontend/src/lib/components/git_sync/GitSyncSection.svelte index 04b23b60c3..2ec6696da5 100644 --- a/frontend/src/lib/components/git_sync/GitSyncSection.svelte +++ b/frontend/src/lib/components/git_sync/GitSyncSection.svelte @@ -12,7 +12,6 @@ import type { GitSyncRepository } from './GitSyncContext.svelte' import GitSyncModalManager from './GitSyncModalManager.svelte' import { enterpriseLicense, workspaceStore, userWorkspaces } from '$lib/stores' - import { base } from '$lib/base' import { WorkspaceService } from '$lib/gen' import { sendUserToast } from '$lib/toast' import { apiErrorMessage } from '$lib/utils' @@ -106,7 +105,7 @@ const devPairingHref = $derived.by(() => { const parent = currentWorkspace?.parent_workspace_id if (!parent || !$userWorkspaces?.some((w) => w.id === parent)) return undefined - return `${base}/workspace_settings?workspace=${parent}&tab=dev_workspace` + return `/workspace_settings?workspace=${parent}&tab=dev_workspace` }) const repositories = $derived(gitSyncContext?.repositories ?? []) diff --git a/frontend/src/lib/components/home/CreateActionsMenu.svelte b/frontend/src/lib/components/home/CreateActionsMenu.svelte index 3193438067..2e22b0765c 100644 --- a/frontend/src/lib/components/home/CreateActionsMenu.svelte +++ b/frontend/src/lib/components/home/CreateActionsMenu.svelte @@ -1,5 +1,4 @@ + +{#snippet header(step: number, title: string, description: string)} +
+ {step} +
+

{title}

+

{description}

+
+
+{/snippet} + +
+
+ Windmill app{ctx.workspace ? ' · ' + ctx.workspace : ''} +

{firstName ? 'Welcome, ' + firstName : 'Welcome'}

+

+ Turn the scripts and flows deployed in this workspace into a tool anyone can use. Edit + App.svelte to make it yours. +

+
+ +

What you can build

+
+ {#each uses as use (use.title)} +
+

{use.title}

+

{use.text}

+
+ {/each} +
+

{${JSON.stringify(BUILDER_SAFETY)}}

+ +

How it works

+
+ {#each steps as step, i (step.title)} +
+ {@render header(i + 1, step.title, step.description)} +
+ {#if step.path} +
+ {#each step.path as label, j (label)} + {#if j > 0}→{/if} + {label} + {/each} +
+ {/if} + {step.note} +
+ {#if step.code} + 
{step.code}
+ {/if} +
+ {/each} + +
+ {@render header(steps.length + 1, 'Know who is viewing', 'Windmill passes the workspace and the viewer to the app.')} +
+ {#if ctx.workspace} +
Workspace {ctx.workspace}
+
+ Viewer + {viewer ? viewer.name || viewer.username : 'Anonymous visitor'} +
+ {:else} + The editor preview has no viewer. Deploy the app to see yours here. + {/if} +
+ 
{contextCode}
+
+
+
+ + +` + +export const BUILDER_FRAMEWORK_TEMPLATES = { + react19: { ...react19Template, '/App.tsx': builderAppTsx }, + react18: { ...react18Template, '/App.tsx': builderAppTsx }, + svelte5: { ...svelte5Template, '/App.svelte': builderAppSvelte } +} + type StarterArg = { type: 'string' | 'number' } function inlineRunnable( diff --git a/frontend/src/lib/components/resources/ResourcesList.svelte b/frontend/src/lib/components/resources/ResourcesList.svelte index 687682fd4f..2b0cc5eb1e 100644 --- a/frontend/src/lib/components/resources/ResourcesList.svelte +++ b/frontend/src/lib/components/resources/ResourcesList.svelte @@ -44,8 +44,8 @@ import { OauthService, ResourceService, WorkspaceService, type ListableResource } from '$lib/gen' import { enterpriseLicense, userWorkspaces } from '$lib/stores' import { sendUserToast } from '$lib/toast' + import { useEditRights } from '$lib/operatorWriteRights' import { - canWrite, classNames, debounce, emptySchema, @@ -96,6 +96,7 @@ const operatingWorkspace = useOperatingWorkspace() const operatingUser = useOperatingUser() + const editRights = useEditRights() const hosted = useHostedPage() type ResourceW = ListableResource & { canWrite: boolean; marked?: string } @@ -334,7 +335,7 @@ const result = (await ResourceService.listResource(apiParams)).map((x) => { return { canWrite: - canWrite(x.path, x.extra_perms!, operatingUser.current) && + editRights.canEditItem('resource', x.path, x.extra_perms) && $operatingWorkspace! == x.workspace_id, ...x } diff --git a/frontend/src/lib/components/runs/DispatchEventsTable.svelte b/frontend/src/lib/components/runs/DispatchEventsTable.svelte index 828437fee6..144a2d52e1 100644 --- a/frontend/src/lib/components/runs/DispatchEventsTable.svelte +++ b/frontend/src/lib/components/runs/DispatchEventsTable.svelte @@ -1,5 +1,4 @@ diff --git a/frontend/src/lib/components/sessions/SessionPicker.svelte b/frontend/src/lib/components/sessions/SessionPicker.svelte index d4f7fa66ab..7e8283e406 100644 --- a/frontend/src/lib/components/sessions/SessionPicker.svelte +++ b/frontend/src/lib/components/sessions/SessionPicker.svelte @@ -66,7 +66,6 @@ import { sendUserToast } from '$lib/toast' import { currentWorkspaceRootId, workspaceRootId } from './sessionScope.svelte' import { page } from '$app/state' - import { base } from '$app/paths' import DevWorkspaceBadge from '$lib/components/DevWorkspaceBadge.svelte' import { dateBucket, @@ -119,7 +118,7 @@ // Only highlight the active session while the sessions page is open — elsewhere // `currentSessionId` lingers but no row should appear selected. - const onSessionsPage = $derived(page.url.pathname.startsWith(`${base}/sessions`)) + const onSessionsPage = $derived(page.url.pathname.startsWith('/sessions')) const sessionActive = $derived(onSessionsPage) interface Props { diff --git a/frontend/src/lib/components/sessions/SessionsBetaBanner.svelte b/frontend/src/lib/components/sessions/SessionsBetaBanner.svelte index 86b6ba030a..0d523a4e65 100644 --- a/frontend/src/lib/components/sessions/SessionsBetaBanner.svelte +++ b/frontend/src/lib/components/sessions/SessionsBetaBanner.svelte @@ -1,5 +1,4 @@ @@ -17,7 +16,7 @@ 1. Choose a trigger script using the script picker that will poll for new events regularly script picker @@ -26,7 +25,7 @@ 2. Add logic to process each new event create action diff --git a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte index 18d5611a67..141e67fd1b 100644 --- a/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte +++ b/frontend/src/lib/components/triggers/schedules/ScheduleEditorInner.svelte @@ -33,9 +33,8 @@ type Schedule, type ErrorHandler } from '$lib/gen' - import { canWrite, emptyString, formatCron, sendUserToast, cronV1toV2 } from '$lib/utils' - import { useScheduleLock } from '$lib/operatorWriteRights' - import { base } from '$lib/base' + import { emptyString, formatCron, sendUserToast, cronV1toV2 } from '$lib/utils' + import { useEditRights } from '$lib/operatorWriteRights' import Section from '$lib/components/Section.svelte' import { List, Loader2, AlertTriangle } from 'lucide-svelte' import autosize from '$lib/autosize' @@ -52,7 +51,7 @@ import PermissionedAsLine from '../PermissionedAsLine.svelte' import { useActingUser } from '$lib/actingUser.svelte' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' - const scheduleLock = useScheduleLock() + const editRights = useEditRights() let { useDrawer = true, @@ -145,7 +144,9 @@ const acting = useActingUser(() => wsId) const actingUser = $derived(acting.current) const can_write = $derived( - (permsPath === undefined || canWrite(permsPath, extraPerms, actingUser)) && !$scheduleLock + permsPath === undefined + ? editRights.roleCanAuthor('schedule') + : editRights.canEditItem('schedule', permsPath, extraPerms) ) // Editing the runnable is closed to operators, and an unresolved acting user is no // evidence that this one isn't. @@ -182,6 +183,7 @@ }, 100) // Do not show loading spinner for the first 100ms drawerLoading = true acting.forgetFailures() + editRights.forgetFailures() saveDraftHandler = undefined try { drawer?.openDrawer() @@ -287,6 +289,7 @@ }, 100) // Do not show loading spinner for the first 100ms drawerLoading = true acting.forgetFailures() + editRights.forgetFailures() try { let s: Schedule | undefined if (schedule_path) { @@ -511,7 +514,6 @@ deploymentLoading = false } - let drawer: Drawer | undefined = $state() let pathC: Path | undefined = $state() @@ -652,7 +654,7 @@ variant="default" startIcon={{ icon: List }} disabled={!allowSchedule || pathError != '' || emptyString(script_path)} - href={`${base}/runs/?schedule_path=${path}&job_trigger_kind=schedule&show_future_jobs=true`} + href={`/runs/?schedule_path=${path}&job_trigger_kind=schedule&show_future_jobs=true`} > View runs diff --git a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte index 0044abbcc1..1ce283392f 100644 --- a/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/sqs/SqsTriggerEditorInner.svelte @@ -9,8 +9,8 @@ import DrawerContent from '$lib/components/common/drawer/DrawerContent.svelte' import Path from '$lib/components/Path.svelte' import { usedTriggerKinds } from '$lib/stores' - import { canWrite, capitalize, emptyString, sendUserToast } from '$lib/utils' - import { useTriggerLock } from '$lib/operatorWriteRights' + import { capitalize, emptyString, sendUserToast } from '$lib/utils' + import { useEditRights } from '$lib/operatorWriteRights' import { withForkConflictRetry } from '$lib/utils/forkConflict' import { Loader2 } from 'lucide-svelte' import Label from '$lib/components/Label.svelte' @@ -43,7 +43,7 @@ useOperatingWorkspace, useOperatingWorkspaceHref } from '$lib/components/operatingWorkspace.svelte' - const triggerLock = useTriggerLock() + const editRights = useEditRights() interface Props { useDrawer?: boolean @@ -109,8 +109,9 @@ // The acting user in the operating workspace arrives asynchronously, and an unknown user // refuses — so the editor stays read-only until the lookup lands, which is the safe answer. const can_write = $derived( - (permsPath === undefined || canWrite(permsPath, permsForWrite ?? {}, actingUser)) && - !$triggerLock + permsPath === undefined + ? editRights.roleCanAuthor('trigger') + : editRights.canEditItem('trigger', permsPath, permsForWrite) ) let drawerLoading = $state(true) let showLoading = $state(false) @@ -165,6 +166,7 @@ saveDraftHandler = undefined // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() + editRights.forgetFailures() let loadingTimeout = setTimeout(() => { showLoading = true }, 100) // Do not show loading spinner for the first 100ms diff --git a/frontend/src/lib/components/triggers/triggers.svelte.ts b/frontend/src/lib/components/triggers/triggers.svelte.ts index cd25328ff6..7fe40d111d 100644 --- a/frontend/src/lib/components/triggers/triggers.svelte.ts +++ b/frontend/src/lib/components/triggers/triggers.svelte.ts @@ -23,14 +23,15 @@ import { type NativeTrigger, type NativeServiceName } from '$lib/gen' -import { enterpriseLicense } from '$lib/stores' +import { enterpriseLicense, userWorkspaces } from '$lib/stores' import { getLightConfig, sortTriggers, updateTriggersCount, type Trigger } from './utils' import { get, type Writable } from 'svelte/store' import type { TriggerType } from './utils' import type { UserExt } from '$lib/stores' import type { ScheduleTrigger } from '../triggers' -import { canWrite, formatCron, generateRandomString } from '$lib/utils' +import { formatCron, generateRandomString } from '$lib/utils' +import { canEditItem } from '$lib/editRights' export class Triggers { #triggers = $state([]) @@ -130,9 +131,14 @@ export class Triggers { updateTriggers( remoteTriggers: any[], type: TriggerType, + workspaceId: string, user: UserExt | undefined = undefined ): number { const currentTriggers = this.#triggers + const operatorSettings = get(userWorkspaces).find( + (w) => w.id === workspaceId + )?.operator_settings + const kind = type === 'schedule' ? 'schedule' : 'trigger' // Identify triggers with draftConfig to preserve const configuredTriggers = currentTriggers.filter( (t) => t.type === type && !t.isDraft && t.draftConfig @@ -151,7 +157,7 @@ export class Triggers { path: trigger.path, isPrimary: type === 'schedule' && trigger.path === trigger.script_path, isDraft: false, - canWrite: canWrite(trigger.path, trigger.extra_perms, user), + canWrite: canEditItem(kind, trigger.path, trigger.extra_perms, user, operatorSettings), draftConfig: draftConfig, lightConfig: getLightConfig(type, trigger) } @@ -199,7 +205,7 @@ export class Triggers { isFlow }) - const scheduleCount = this.updateTriggers(allDeployedSchedules, 'schedule', user) + const scheduleCount = this.updateTriggers(allDeployedSchedules, 'schedule', workspaceId, user) const updatedPrimarySchedule = this.#triggers.find((s) => s.isPrimary) triggersCountStore.update((triggersCount) => { return { @@ -234,7 +240,7 @@ export class Triggers { path, isFlow }) - const wsCount = this.updateTriggers(wsTriggers, 'websocket', user) + const wsCount = this.updateTriggers(wsTriggers, 'websocket', workspaceId, user) triggersCountStore.update((triggersCount) => { return { ...(triggersCount ?? {}), @@ -260,7 +266,7 @@ export class Triggers { path, isFlow }) - const pgCount = this.updateTriggers(pgTriggers, 'postgres', user) + const pgCount = this.updateTriggers(pgTriggers, 'postgres', workspaceId, user) triggersCountStore.update((triggersCount) => { return { ...(triggersCount ?? {}), @@ -286,7 +292,7 @@ export class Triggers { path, isFlow }) - const kafkaCount = this.updateTriggers(kafkaTriggers, 'kafka', user) + const kafkaCount = this.updateTriggers(kafkaTriggers, 'kafka', workspaceId, user) triggersCountStore.update((triggersCount) => { return { ...(triggersCount ?? {}), @@ -312,7 +318,7 @@ export class Triggers { path, isFlow }) - const natsCount = this.updateTriggers(natsTriggers, 'nats', user) + const natsCount = this.updateTriggers(natsTriggers, 'nats', workspaceId, user) triggersCountStore.update((triggersCount) => { return { ...(triggersCount ?? {}), @@ -338,7 +344,7 @@ export class Triggers { path, isFlow }) - const mqttCount = this.updateTriggers(mqttTriggers, 'mqtt', user) + const mqttCount = this.updateTriggers(mqttTriggers, 'mqtt', workspaceId, user) triggersCountStore.update((triggersCount) => { return { ...(triggersCount ?? {}), @@ -364,7 +370,7 @@ export class Triggers { path, isFlow }) - const amqpCount = this.updateTriggers(amqpTriggers, 'amqp', user) + const amqpCount = this.updateTriggers(amqpTriggers, 'amqp', workspaceId, user) triggersCountStore.update((triggersCount) => { return { ...(triggersCount ?? {}), @@ -390,7 +396,7 @@ export class Triggers { path, isFlow }) - const sqsCount = this.updateTriggers(sqsTriggers, 'sqs', user) + const sqsCount = this.updateTriggers(sqsTriggers, 'sqs', workspaceId, user) triggersCountStore.update((triggersCount) => { return { ...(triggersCount ?? {}), @@ -416,7 +422,7 @@ export class Triggers { path, isFlow }) - const gcpCount = this.updateTriggers(gcpTriggers, 'gcp', user) + const gcpCount = this.updateTriggers(gcpTriggers, 'gcp', workspaceId, user) triggersCountStore.update((triggersCount) => { return { ...(triggersCount ?? {}), @@ -442,7 +448,7 @@ export class Triggers { path, isFlow }) - const azureCount = this.updateTriggers(azureTriggers, 'azure', user) + const azureCount = this.updateTriggers(azureTriggers, 'azure', workspaceId, user) triggersCountStore.update((triggersCount) => { return { ...(triggersCount ?? {}), @@ -468,7 +474,7 @@ export class Triggers { path, isFlow }) - const httpCount = this.updateTriggers(httpTriggers, 'http', user) + const httpCount = this.updateTriggers(httpTriggers, 'http', workspaceId, user) triggersCountStore.update((triggersCount) => { return { ...(triggersCount ?? {}), @@ -494,7 +500,7 @@ export class Triggers { path, isFlow }) - const emailCount = this.updateTriggers(emailTriggers, 'email', user) + const emailCount = this.updateTriggers(emailTriggers, 'email', workspaceId, user) triggersCountStore.update((triggersCount) => { return { ...(triggersCount ?? {}), @@ -530,7 +536,7 @@ export class Triggers { extra_perms: {}, service_config: t.service_config })) - const count = this.updateTriggers(triggerData, serviceName, user) + const count = this.updateTriggers(triggerData, serviceName, workspaceId, user) const countProperty = `${serviceName}_count` triggersCountStore.update((triggersCount) => { return { diff --git a/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte b/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte index 9033aab3cf..218430316a 100644 --- a/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte +++ b/frontend/src/lib/components/triggers/webhook/WebhooksConfigSection.svelte @@ -16,7 +16,7 @@ import ClipboardPanel from '../../details/ClipboardPanel.svelte' import { copyToClipboard, isObject, readFieldsRecursively } from '$lib/utils' // import { page } from '$app/state' - import { base } from '$lib/base' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' import TriggerTokens from '../TriggerTokens.svelte' import { userStore } from '$lib/stores' import UserSettings from '../../UserSettings.svelte' @@ -46,7 +46,7 @@ const operatingWorkspace = useOperatingWorkspace() const wsId = $derived($operatingWorkspace) - const WEBHOOK_BASE_URL = $derived(`${location.origin}${base}/api/w/${wsId}/jobs`) + const WEBHOOK_BASE_URL = $derived(`${apiBaseUrl()}/api/w/${wsId}/jobs`) let baseWebhookUrl = $derived.by(() => { let webhookUrlPath: string @@ -223,9 +223,7 @@ async function triggerJob() { function waitForJobCompletion(UUID) { return new Promise(async (resolve, reject) => { try { - const endpoint = \`${ - location.origin - }/api/w/${wsId}/jobs_u/completed/get_result_maybe/\${UUID}\`; + const endpoint = \`${apiBaseUrl()}/api/w/${wsId}/jobs_u/completed/get_result_maybe/\${UUID}\`; const checkResponse = await fetch(endpoint, { method: 'GET', headers: ${JSON.stringify(headers(), null, 2).replaceAll('\n', '\n\t\t\t\t')} @@ -268,7 +266,7 @@ ${ ? 'echo -E $RESULT | jq' : requestType === 'async' ? ` -URL="${location.origin}/api/w/${wsId}/jobs_u/completed/get_result_maybe/$UUID" +URL="${apiBaseUrl()}/api/w/${wsId}/jobs_u/completed/get_result_maybe/$UUID" while true; do curl -s -H "Authorization: Bearer $TOKEN" $URL -o res.json COMPLETED=$(cat res.json | jq .completed) diff --git a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte index 007bc39e47..8cfe3480fe 100644 --- a/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte +++ b/frontend/src/lib/components/triggers/websocket/WebsocketTriggerEditorInner.svelte @@ -25,8 +25,8 @@ type TriggerMode } from '$lib/gen' import { usedTriggerKinds } from '$lib/stores' - import { canWrite, emptySchema, emptyString, sendUserToast } from '$lib/utils' - import { useTriggerLock } from '$lib/operatorWriteRights' + import { emptySchema, emptyString, sendUserToast } from '$lib/utils' + import { useEditRights } from '$lib/operatorWriteRights' import { withForkConflictRetry } from '$lib/utils/forkConflict' import Section from '$lib/components/Section.svelte' import { Loader2, X, Plus } from 'lucide-svelte' @@ -59,7 +59,7 @@ useOperatingWorkspace, useOperatingWorkspaceHref } from '$lib/components/operatingWorkspace.svelte' - const triggerLock = useTriggerLock() + const editRights = useEditRights() interface Props { useDrawer?: boolean @@ -137,8 +137,9 @@ // The acting user in the operating workspace arrives asynchronously, and an unknown user // refuses — so the editor stays read-only until the lookup lands, which is the safe answer. const can_write = $derived( - (permsPath === undefined || canWrite(permsPath, permsForWrite ?? {}, actingUser)) && - !$triggerLock + permsPath === undefined + ? editRights.roleCanAuthor('trigger') + : editRights.canEditItem('trigger', permsPath, permsForWrite) ) let drawerLoading = $state(true) let showLoading = $state(false) @@ -206,6 +207,7 @@ ) { // A `whoami` that failed earlier would otherwise pin this workspace to "unknown user". operatingUser.forgetFailures() + editRights.forgetFailures() let loadingTimeout = setTimeout(() => { showLoading = true }, 100) // Do not show loading spinner for the first 100ms diff --git a/frontend/src/lib/components/variables/VariablesList.svelte b/frontend/src/lib/components/variables/VariablesList.svelte index 66f293a11d..3805b51bbb 100644 --- a/frontend/src/lib/components/variables/VariablesList.svelte +++ b/frontend/src/lib/components/variables/VariablesList.svelte @@ -38,7 +38,8 @@ import type { ContextualVariable, ListableVariable, WorkspaceDeployUISettings } from '$lib/gen' import { enterpriseLicense, userWorkspaces } from '$lib/stores' import { sendUserToast } from '$lib/toast' - import { canWrite, isOwner, truncate } from '$lib/utils' + import { isOwner, truncate } from '$lib/utils' + import { useEditRights } from '$lib/operatorWriteRights' import { isDeployable, ALL_DEPLOYABLE } from '$lib/utils_deployable' import { Plus, @@ -64,6 +65,7 @@ const operatingWorkspace = useOperatingWorkspace() const operatingUser = useOperatingUser() + const editRights = useEditRights() const hosted = useHostedPage() function editVariable(path: string) { @@ -205,7 +207,7 @@ const result = (await VariableService.listVariable(apiParams)).map((x) => { return { canWrite: - canWrite(x.path, x.extra_perms!, operatingUser.current) && + editRights.canEditItem('variable', x.path, x.extra_perms) && x.workspace_id == $operatingWorkspace, ...x } diff --git a/frontend/src/lib/components/wizards/AppPicker.svelte b/frontend/src/lib/components/wizards/AppPicker.svelte index 17f384c71d..7e8a1e91a6 100644 --- a/frontend/src/lib/components/wizards/AppPicker.svelte +++ b/frontend/src/lib/components/wizards/AppPicker.svelte @@ -2,18 +2,14 @@ import { getContext, onMount } from 'svelte' import DarkModeObserver from '../DarkModeObserver.svelte' import { AppService, type ListableApp } from '$lib/gen' - import { canWrite } from '$lib/utils' + import { useEditRights } from '$lib/operatorWriteRights' import type { AppViewerContext } from '../apps/types' import Alert from '../common/alert/Alert.svelte' import Select from '../select/Select.svelte' - import { - useOperatingUser, - useOperatingWorkspace - } from '$lib/components/operatingWorkspace.svelte' + import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' const operatingWorkspace = useOperatingWorkspace() - const operatingUser = useOperatingUser() - const actingUser = $derived(operatingUser.current) + const editRights = useEditRights() interface Props { value?: string @@ -32,9 +28,8 @@ ).map((app: ListableApp) => { return { canWrite: - canWrite(app.path!, app.extra_perms!, actingUser) && - app.workspace_id == $operatingWorkspace && - !actingUser?.operator, + editRights.canEditItem('app', app.path!, app.extra_perms) && + app.workspace_id == $operatingWorkspace, ...app } }) diff --git a/frontend/src/lib/components/workspacePicker.ts b/frontend/src/lib/components/workspacePicker.ts index 61b4d3239e..c33ad36843 100644 --- a/frontend/src/lib/components/workspacePicker.ts +++ b/frontend/src/lib/components/workspacePicker.ts @@ -19,6 +19,8 @@ export type WorkspaceItem = { * so read it with `draftOnly` to tell "nothing deployed" from "deployed, with * edits pending". */ hasDraft?: boolean + /** The item's own sharing, which `canEditItem` reads with the folder rules. */ + extraPerms?: Record } /** Path to display (and group by) in pickers: the friendly draft path when @@ -140,7 +142,8 @@ export async function loadKind( kind: 'flow' as const, draftPath: f.draft_path, draftOnly: f.draft_only ?? false, - hasDraft: f.is_draft ?? false + hasDraft: f.is_draft ?? false, + extraPerms: f.extra_perms })) } else if (kind === 'script') { const scripts = await ScriptService.listScripts({ @@ -154,7 +157,8 @@ export async function loadKind( kind: 'script' as const, draftPath: s.draft_path, draftOnly: s.draft_only ?? false, - hasDraft: s.is_draft ?? false + hasDraft: s.is_draft ?? false, + extraPerms: s.extra_perms })) } else { const apps = await AppService.listApps({ @@ -168,7 +172,8 @@ export async function loadKind( raw_app: a.raw_app ?? false, draftPath: a.draft_path, draftOnly: a.draft_only ?? false, - hasDraft: a.is_draft ?? false + hasDraft: a.is_draft ?? false, + extraPerms: a.extra_perms })) } // Only commit if the cache version hasn't changed since we started — diff --git a/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte index 5ae952f03e..959b666363 100644 --- a/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte +++ b/frontend/src/lib/components/workspaceSettings/CreateWorkspaceInner.svelte @@ -64,7 +64,6 @@ import TextInput from '../text_input/TextInput.svelte' import { jobManager } from '$lib/services/JobManager' import Alert from '../common/alert/Alert.svelte' - import { base } from '$lib/base' import Label from '../Label.svelte' import ForkDatatableSection from './ForkDatatableSection.svelte' import Select from '../select/Select.svelte' @@ -1106,7 +1105,7 @@ disabled={forkCreationLoading} variant="default" size="sm" - href="{base}/user/workspaces">← Back to workspaces← Back to workspaces {/if} {#if !forkCreationLoading} diff --git a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte index f6929e0779..a01318d916 100644 --- a/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte +++ b/frontend/src/lib/components/workspaceSettings/DucklakeSettings.svelte @@ -91,7 +91,6 @@ import type { GetSettingsResponse } from '$lib/gen' import { enterpriseLicense, superadmin, userWorkspaces, workspaceStore } from '$lib/stores' - import { base } from '$app/paths' import Toggle from '../Toggle.svelte' import { sendUserToast } from '$lib/toast' import ExploreAssetButton from '../ExploreAssetButton.svelte' @@ -653,7 +652,7 @@ {/if} {#if ducklakeSavedSettings.ducklakes.find((d) => d.name === ducklake.name)?.maintenance?.enabled} ) { diff --git a/frontend/src/lib/components/workspaceSettings/supabaseOauth.svelte.ts b/frontend/src/lib/components/workspaceSettings/supabaseOauth.svelte.ts index 3e11ec2b9d..94c21cf591 100644 --- a/frontend/src/lib/components/workspaceSettings/supabaseOauth.svelte.ts +++ b/frontend/src/lib/components/workspaceSettings/supabaseOauth.svelte.ts @@ -1,9 +1,8 @@ import { fromStore } from 'svelte/store' -import { base } from '$lib/base' import { oauthStore } from '$lib/stores' const OAUTH_WINDOW = 'windmill_supabase_oauth' -const CONNECT_URL = `${base}/api/oauth/connect/supabase_wizard` +const CONNECT_URL = '/api/oauth/connect/supabase_wizard' /** * The Supabase authorization leg, driven from a popup. diff --git a/frontend/src/lib/components/workspaceSettings/supabaseProvisioning.ts b/frontend/src/lib/components/workspaceSettings/supabaseProvisioning.ts index 5040b5ab38..ee91a0ba3c 100644 --- a/frontend/src/lib/components/workspaceSettings/supabaseProvisioning.ts +++ b/frontend/src/lib/components/workspaceSettings/supabaseProvisioning.ts @@ -7,7 +7,6 @@ */ import { DEFAULT_SSLMODE } from '$lib/utils/postgresConnectionString' -import { base } from '$lib/base' import { oauthStore } from '$lib/stores' import { get } from 'svelte/store' @@ -90,19 +89,19 @@ export function supabaseErrorMessage(body: string): string { } export async function listSupabaseOrgs(token: string): Promise { - const res = await fetch(`${base}/api/oauth/list_supabase_orgs`, { headers: headers(token) }) + const res = await fetch('/api/oauth/list_supabase_orgs', { headers: headers(token) }) return unwrap(res, 'Could not list your Supabase organizations') } export async function listSupabaseProjects(token: string): Promise { - const res = await fetch(`${base}/api/oauth/list_supabase`, { headers: headers(token) }) + const res = await fetch('/api/oauth/list_supabase', { headers: headers(token) }) return unwrap(res, 'Could not list your Supabase projects') } /** Plan of one organization, which the list endpoint does not carry. */ export async function getSupabaseOrgPlan(token: string, slug: string): Promise { try { - const res = await fetch(`${base}/api/oauth/get_supabase_org/${slug}`, { + const res = await fetch(`/api/oauth/get_supabase_org/${slug}`, { headers: headers(token) }) if (!res.ok) return undefined @@ -132,7 +131,7 @@ export async function createSupabaseProject( token: string, args: { name: string; organizationSlug: string; region: string; dbPass: string } ): Promise { - const res = await fetch(`${base}/api/oauth/create_supabase_project`, { + const res = await fetch('/api/oauth/create_supabase_project', { method: 'POST', headers: headers(token), body: JSON.stringify({ @@ -185,7 +184,7 @@ export async function waitUntilSupabaseHealthy( * that landed on another one, and the resulting resource never connects. */ export async function getSupabasePooler(token: string, projectId: string): Promise { - const res = await fetch(`${base}/api/oauth/get_supabase_pooler/${projectId}`, { + const res = await fetch(`/api/oauth/get_supabase_pooler/${projectId}`, { headers: headers(token) }) const configs: SupabasePooler[] = await unwrap(res, 'Could not read the connection details') diff --git a/frontend/src/lib/draftAddRedirect.ts b/frontend/src/lib/draftAddRedirect.ts index 242893f782..2c027ac1da 100644 --- a/frontend/src/lib/draftAddRedirect.ts +++ b/frontend/src/lib/draftAddRedirect.ts @@ -1,5 +1,4 @@ import { redirect } from '@sveltejs/kit' -import { base } from '$app/paths' import { mintDraftPath } from '$lib/mintDraftPath' /** @@ -8,7 +7,7 @@ import { mintDraftPath } from '$lib/mintDraftPath' * * Mints a fresh `u//draft_` path (via the shared `mintDraftPath`, * so the SDK builder wrappers stay in lockstep) and 307s to - * `{base}/{editPrefix}/?new_draft=true&`. `new_draft=true` + * `/{editPrefix}/?new_draft=true&`. `new_draft=true` * tells the edit route to seed an empty editor instead of 404-ing. The hash is * carried over too — fork / handler-template buttons encode a payload into it * that the edit route's `new_draft` branch consumes. @@ -23,6 +22,6 @@ export function makeDraftAddLoad(editPrefix: string) { // points at the PREVIOUS page, but every hash-payload producer arrives // as a full page load, so the hash is correct. const hash = typeof window !== 'undefined' ? window.location.hash : '' - redirect(307, `${base}/${editPrefix}/${path}?${params.toString()}${hash}`) + redirect(307, `/${editPrefix}/${path}?${params.toString()}${hash}`) } } diff --git a/frontend/src/lib/editRights.test.ts b/frontend/src/lib/editRights.test.ts new file mode 100644 index 0000000000..faa853e708 --- /dev/null +++ b/frontend/src/lib/editRights.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest' +import type { UserExt } from '$lib/stores' +import { canEditItem, roleCanAuthor, roleCanDraft } from './editRights' + +function user(overrides: Partial): UserExt { + return { + email: 'op@windmill.dev', + username: 'op', + name: 'op', + is_admin: false, + is_super_admin: false, + created_at: '', + groups: [], + pgroups: [], + folders: [], + folders_owners: [], + folders_read: [], + operator: true, + ...overrides + } as UserExt +} + +describe('edit rights', () => { + it('lets a builder operator author flows and nothing else', () => { + const builder = { builder_flows: true } + expect(roleCanAuthor('flow', user({}), builder)).toBe(true) + expect(roleCanAuthor('script', user({}), builder)).toBe(false) + expect(roleCanAuthor('flow', user({}), undefined)).toBe(false) + expect(roleCanDraft('raw_app', user({}), builder)).toBe(false) + expect(roleCanDraft('raw_app', user({}), { builder_apps: true })).toBe(true) + }) + + // drafts.rs admits an admin before its operator branch; the item handlers do not. + it('lets an admin who is also an operator save drafts, but not author code', () => { + const adminOperator = user({ is_admin: true }) + expect(roleCanDraft('script', adminOperator, undefined)).toBe(true) + expect(roleCanAuthor('script', adminOperator, undefined)).toBe(false) + }) + + // drafts.rs refuses an operator every draft but a builder flow, rights to write directly or not. + it('refuses an operator drafts of what they may write directly', () => { + expect(roleCanAuthor('trigger', user({}), undefined)).toBe(true) + expect(roleCanDraft('trigger', user({}), undefined)).toBe(false) + expect(roleCanDraft('resource', user({}), undefined)).toBe(false) + }) + + // The role alone is not enough: a builder still cannot edit a flow in a folder they only read. + it('requires write access to the item on top of the role', () => { + const builder = { builder_flows: true } + expect(canEditItem('flow', 'f/demo/sum', {}, user({}), builder)).toBe(false) + expect(canEditItem('flow', 'u/op/sum', {}, user({}), builder)).toBe(true) + }) +}) diff --git a/frontend/src/lib/editRights.ts b/frontend/src/lib/editRights.ts new file mode 100644 index 0000000000..9073a03d28 --- /dev/null +++ b/frontend/src/lib/editRights.ts @@ -0,0 +1,115 @@ +import type { OperatorSettings } from '$lib/gen' +import type { UserExt } from '$lib/stores' +import { canWrite } from '$lib/utils' + +/** + * The frontend's one copy of "may this user edit this item". The server decides — drafts.rs + * `require_can_write_path`, the item handlers' operator refusals, `gate_operator_writes`, and + * row-level security — and this mirrors it so a control the server would refuse is never + * offered. Check edits through `canEditItem` (or `useEditRights` in a component), never by + * pairing `canWrite` with an operator test of your own. + */ + +/** Kinds whose role half differs. Any other kind is code an operator never authors. */ +export type EditKind = + | 'script' + | 'flow' + | 'app' + | 'raw_app' + | 'agent' + | 'pipeline' + | 'schedule' + | 'trigger' + | 'resource' + | 'variable' + | 'group' + | (string & {}) + +type OperatorRights = + | Pick< + NonNullable, + 'builder_flows' | 'builder_apps' | 'manage_schedules' | 'manage_triggers' + > + | null + | undefined + +/** + * Whether a user's role lets them author items of `kind` — create, edit, deploy or delete one — + * in a workspace with these `operator_settings`: the role half of every check against the item + * handlers. Saving a draft is `roleCanDraft`'s rule, and which items they may write is + * `canEditItem`'s other half. For an operator: + * - code (scripts, low-code apps, agents, pipelines): never; + * - flows and full-code apps: where the workspace grants `builder_flows` / `builder_apps` + * (`check_operator_can_build`); + * - schedules and triggers: unless the workspace set `manage_*` to false — a withdrawable + * right, so an unset key grants it (docs/operator-write-rights.md); + * - resources, variables and groups: always, their permissions alone decide. + */ +export function roleCanAuthor( + kind: EditKind, + user: Pick | undefined, + operatorSettings: OperatorRights +): boolean { + if (!user?.operator) return true + switch (kind) { + case 'flow': + case 'raw_app': + return builderGrants(kind, operatorSettings) + case 'schedule': + return operatorSettings?.manage_schedules !== false + case 'trigger': + return operatorSettings?.manage_triggers !== false + case 'resource': + case 'variable': + case 'group': + return true + default: + return false + } +} + +/** + * Whether the user may save drafts of `kind` — what every session editor writes. Mirrors drafts.rs + * `require_can_write_path`, which differs from the item handlers both ways: it admits an admin + * before its operator branch, and it refuses every operator draft but a flow or full-code app + * under its builder right — schedules, triggers, resources and variables included, though an operator may + * write those directly. + */ +export function roleCanDraft( + kind: EditKind, + user: Pick | undefined, + operatorSettings: OperatorRights +): boolean { + if (user?.is_admin || user?.is_super_admin || !user?.operator) return true + return builderGrants(kind, operatorSettings) +} + +function builderGrants(kind: EditKind, operatorSettings: OperatorRights): boolean { + return ( + (kind === 'flow' && operatorSettings?.builder_flows === true) || + (kind === 'raw_app' && operatorSettings?.builder_apps === true) + ) +} + +/** `roleCanDraft` for the item at `path`, with its own permissions as in `canEditItem`. */ +export function canDraftItem( + kind: EditKind, + path: string, + extraPerms: Record | undefined, + user: UserExt | undefined, + operatorSettings: OperatorRights +): boolean { + return roleCanDraft(kind, user, operatorSettings) && canWrite(path, extraPerms ?? {}, user) +} + +/** Whether `user` may edit the item at `path`: their role authors `kind`, and the item's own + * permissions (`canWrite`: ownership, folder write, `extra_perms`) let them write it. */ +export function canEditItem( + kind: EditKind, + path: string, + extraPerms: Record | undefined, + user: UserExt | undefined, + operatorSettings: OperatorRights +): boolean { + return roleCanAuthor(kind, user, operatorSettings) && canWrite(path, extraPerms ?? {}, user) +} diff --git a/frontend/src/lib/githubApp.ts b/frontend/src/lib/githubApp.ts index 666734546e..0c19cd8de2 100644 --- a/frontend/src/lib/githubApp.ts +++ b/frontend/src/lib/githubApp.ts @@ -4,7 +4,6 @@ import { type GetGhesConfigResponse } from '$lib/gen' import { sendUserToast } from '$lib/toast' -import { base } from '$lib/base' export interface GitHubAppState { loadingGithubInstallations: boolean @@ -137,7 +136,7 @@ export async function loadGithubInstallations( const stateParam = encodeURIComponent( JSON.stringify({ workspace_id: currentWorkspace, - base_url: window.location.origin + base + base_url: window.location.origin }) ) diff --git a/frontend/src/lib/instanceUi.ts b/frontend/src/lib/instanceUi.ts index b9b87d139c..3a7bb1b99c 100644 --- a/frontend/src/lib/instanceUi.ts +++ b/frontend/src/lib/instanceUi.ts @@ -2,7 +2,8 @@ import { writable } from 'svelte/store' import type { GetInstanceUiResponse } from '$lib/gen' /** - * The last `/settings/instance_ui` response: the announcement banner and accent color. + * The last `/settings/instance_ui` response: the announcement banner, accent color + * and API base url. * `undefined` until one lands, and left as is when a fetch fails, so a transient error * never retracts an announcement still in force. Written only by `InstanceUiSync`. */ diff --git a/frontend/src/lib/logout.test.ts b/frontend/src/lib/logout.test.ts index 160bce9934..5bdc15bac3 100644 --- a/frontend/src/lib/logout.test.ts +++ b/frontend/src/lib/logout.test.ts @@ -12,9 +12,25 @@ import { clearUser } from './logout' describe('clearUser', () => { beforeEach(() => { sessionStorage.clear() + localStorage.clear() vi.clearAllMocks() }) + it('drops credential-bearing localStorage entries and keeps preferences', async () => { + const credentialKeys = ['oauth-callback', 'mcp-oauth-callback', 'test_dev_token'] + for (const key of credentialKeys) { + localStorage.setItem(key, 'a-token') + } + localStorage.setItem('dark-mode', 'true') + + await clearUser() + + for (const key of credentialKeys) { + expect(localStorage.getItem(key)).toBeNull() + } + expect(localStorage.getItem('dark-mode')).toBe('true') + }) + // Logging out while impersonating must not leave the impersonator's own token behind. it('drops the impersonation token backup', async () => { sessionStorage.setItem('pre_impersonation_token', 'a-token') diff --git a/frontend/src/lib/logout.ts b/frontend/src/lib/logout.ts index 58fce16fe3..b4782badd3 100644 --- a/frontend/src/lib/logout.ts +++ b/frontend/src/lib/logout.ts @@ -6,15 +6,19 @@ import { clearStores } from './storeUtils' // Note: logout and logoutWithRedirect have been moved to logoutKit.ts // as they depend on SvelteKit navigation -// The impersonation backup holds the impersonator's own auth token, so it must go on -// logout too, not only when impersonation is ended from the banner — otherwise a usable -// token stays readable for the lifetime of the tab. -function clearImpersonationFromStorage(): void { +// Every browser-storage entry that can hold a credential must go on logout. The impersonation +// backup is the impersonator's own auth token; `oauth-callback` and `mcp-oauth-callback` carry a +// connected account's token response when the OAuth popup had no opener to post it to; +// `test_dev_token` is the session token of the /test_dev pages. +function clearCredentialsFromStorage(): void { try { sessionStorage.removeItem('pre_impersonation_token') sessionStorage.removeItem('pre_impersonation_email') + localStorage.removeItem('oauth-callback') + localStorage.removeItem('mcp-oauth-callback') + localStorage.removeItem('test_dev_token') } catch (e) { - console.error('error interacting with session storage', e) + console.error('error interacting with browser storage', e) } } @@ -23,7 +27,7 @@ export async function clearUser() { noteSessionEmail(undefined) accountSetup.reset() clearStores() - clearImpersonationFromStorage() + clearCredentialsFromStorage() await UserService.logout() } catch (error) {} } diff --git a/frontend/src/lib/navigation.test.ts b/frontend/src/lib/navigation.test.ts index f499ddf1f7..d7f503b4eb 100644 --- a/frontend/src/lib/navigation.test.ts +++ b/frontend/src/lib/navigation.test.ts @@ -2,11 +2,10 @@ import { describe, it, expect, vi } from 'vitest' const goto = vi.hoisted(() => vi.fn(async (_path: string) => {})) vi.mock('$app/navigation', () => ({ goto: (path: string) => goto(path) })) -vi.mock('$app/paths', () => ({ base: '' })) import { buildFilterUrl, setQuery } from './navigation' -// Parse the (un-based) app path the builder returns against a dummy origin so we can +// Parse the app path the builder returns against a dummy origin so we can // assert on pathname / searchParams / hash without caring about ordering. function parse(appPath: string): URL { return new URL(appPath, 'http://x') diff --git a/frontend/src/lib/navigation.ts b/frontend/src/lib/navigation.ts index db8b9f3a80..164b9402c1 100644 --- a/frontend/src/lib/navigation.ts +++ b/frontend/src/lib/navigation.ts @@ -1,23 +1,14 @@ -import { goto as svelteGoto } from '$app/navigation' -import { base as svelteBase } from '$app/paths' +import { goto } from '$app/navigation' import { serializeParam } from '$lib/utils/serializeParam' -export function goto(path: string, options = {}) { - if (svelteBase == '' || path.startsWith('?')) { - return svelteGoto(path, options) - } else { - const fullPath = path.startsWith(svelteBase) ? path : `${svelteBase}${path}` - return svelteGoto(fullPath, options) - } -} +export { goto } /** * Build an in-app deep-link to `pathname` with query-param filters, encoded exactly * as the pages write them (via `serializeParam`) so `useUrlSyncedFilterInstance` * round-trips them back into filter state. Nullish/empty values are dropped; when * `validKeys` is provided, unknown keys are dropped too (structured output guarantees - * shape, not truth). Returns an un-prefixed app path — pass it to `goto`, which adds - * the SvelteKit base. + * shape, not truth). Returns an app path to pass to `goto`. */ export function buildFilterUrl( pathname: string, diff --git a/frontend/src/lib/operatorWriteRights.ts b/frontend/src/lib/operatorWriteRights.ts index 21a27d87f0..916e39dce6 100644 --- a/frontend/src/lib/operatorWriteRights.ts +++ b/frontend/src/lib/operatorWriteRights.ts @@ -1,48 +1,110 @@ -import { derived, type Readable } from 'svelte/store' -import { userWorkspaces, workspaceStore } from '$lib/stores' +import { derived, fromStore, get, readable, type Readable } from 'svelte/store' +import { userWorkspaces, usersWorkspaceStore, workspaceStore } from '$lib/stores' import { useOperatingWorkspace } from '$lib/components/operatingWorkspace.svelte' +import { useActingUser } from '$lib/actingUser.svelte' +import { canDraftItem, canEditItem, roleCanAuthor, roleCanDraft } from '$lib/editRights' + +/** + * The `editRights` rules (`roleCanAuthor` / `canEditItem` for the item handlers, `roleCanDraft` / + * `canDraftItem` for drafts) answered for the user acting in the operating workspace, + * with that workspace's `operator_settings` — or in `workspace`, for a component that is handed + * one rather than sitting under a host that declares it. Reads context and registers an effect: + * call during component initialisation. + */ +export function useEditRights(workspace?: () => string | undefined) { + const operating = fromStore(useOperatingWorkspace()) + const ws = workspace ?? (() => operating.current) + const user = useActingUser(ws) + const workspaces = fromStore(userWorkspaces) + const settings = () => workspaces.current.find((w) => w.id === ws())?.operator_settings + return { + roleCanAuthor: (kind: string) => roleCanAuthor(kind, user.current, settings()), + canEditItem: (kind: string, path: string, extraPerms: Record | undefined) => + canEditItem(kind, path, extraPerms, user.current, settings()), + roleCanDraft: (kind: string) => roleCanDraft(kind, user.current, settings()), + canDraftItem: (kind: string, path: string, extraPerms: Record | undefined) => + canDraftItem(kind, path, extraPerms, user.current, settings()), + /** This hook looks the user up on its own, so an editor that retries a failed lookup on + * its acting user (`forgetFailures`) must retry this one too, or it stays read-only. */ + forgetFailures: () => user.forgetFailures() + } +} /** * Why writes of this kind are locked in the operating workspace, or `undefined` when they are not — * the shape `title` and `disabled` both want. See `docs/operator-write-rights.md`. * - * Only `false` locks. A workspace that never configured the key and a non-operator (whose - * `operator_settings` is null) both hold the right, so `=== true` here would disable the controls - * for everyone. + * The role half of `roleCanAuthor`, phrased for the controls that explain it. `operator_settings` + * is non-null exactly for an operator (`isOperatorInWorkspace`), so it stands in for the user. */ -function writeLock(key: 'manage_schedules' | 'manage_triggers', noun: string) { +function writeLock(kind: 'schedule' | 'trigger', noun: string) { return derived([userWorkspaces, useOperatingWorkspace()], ([$userWorkspaces, $workspace]) => { const settings = $userWorkspaces.find((w) => w.id === $workspace)?.operator_settings // Worded as the server words its refusal of the same write. - return settings?.[key] === false - ? `Operators cannot manage ${noun} in this workspace` - : undefined + return roleCanAuthor(kind, { operator: settings != null }, settings) + ? undefined + : `Operators cannot manage ${noun} in this workspace` }) } /** Reads context: call during component initialisation. */ -export const useScheduleLock = () => writeLock('manage_schedules', 'schedules') +export const useScheduleLock = () => writeLock('schedule', 'schedules') /** Reads context: call during component initialisation. */ -export const useTriggerLock = () => writeLock('manage_triggers', 'triggers') +export const useTriggerLock = () => writeLock('trigger', 'triggers') /** * True when the user is an operator of `workspace` and it granted operators the right to compose - * flows out of runnables that are already deployed. They still author no code, and everywhere else - * `operator` keeps meaning read-only, so a gate on the operator role has to consult this before + * this kind out of runnables that are already deployed. They still author no code, and everywhere + * else `operator` keeps meaning read-only, so a gate on the operator role has to consult this before * refusing. `operator_settings` is null for a non-operator, so this is false for them. + * + * The two rights are granted independently: anything that authors one kind must gate on that + * kind, or a flows-only workspace offers app affordances the backend then refuses. */ -function builderFlows(workspace: Readable): Readable { +function builderRight( + key: 'builder_flows' | 'builder_apps', + workspace: Readable +): Readable { return derived( [userWorkspaces, workspace], ([$userWorkspaces, $workspace]) => - $userWorkspaces.find((w) => w.id === $workspace)?.operator_settings?.builder_flows === true + $userWorkspaces.find((w) => w.id === $workspace)?.operator_settings?.[key] === true + ) +} + +/** Either right, only for surfaces that are not per-kind, such as the create menu's visibility. */ +function anyBuilderRight(workspace: Readable): Readable { + return derived( + [builderRight('builder_flows', workspace), builderRight('builder_apps', workspace)], + ([flows, apps]) => flows || apps ) } /** Builder rights in the operating workspace. Reads context: call during component * initialisation. */ -export const useOperatorBuilderFlows = () => builderFlows(useOperatingWorkspace()) +export const useOperatorBuilderFlows = () => builderRight('builder_flows', useOperatingWorkspace()) +/** Reads context: call during component initialisation. */ +export const useOperatorBuilderApps = () => builderRight('builder_apps', useOperatingWorkspace()) +/** Reads context: call during component initialisation. */ +export const useOperatorBuilderRights = () => anyBuilderRight(useOperatingWorkspace()) /** Builder rights in the navigation workspace, for code outside any component: the legacy AI * chat is one instance for the whole app and acts on the workspace the nav is on. */ -export const navigationOperatorBuilderFlows = builderFlows(workspaceStore) +export const navigationOperatorBuilderFlows = builderRight('builder_flows', workspaceStore) +export const navigationOperatorBuilderApps = builderRight('builder_apps', workspaceStore) +export const navigationOperatorBuilderRights = anyBuilderRight(workspaceStore) + +/** The app builder right in a given workspace, read once: for code acting on an explicit + * workspace rather than the nav's, such as the global AI chat. */ +export const operatorBuilderAppsIn = (workspace: string | undefined) => + get(builderRight('builder_apps', readable(workspace))) + +/** Resolves once the workspace list has loaded. Every builder right reads false until then, so + * code that picks what a builder starts with on a fresh page load has to wait for it. */ +export async function workspaceListLoaded(): Promise { + let unsubscribe = () => {} + await new Promise((resolve) => { + unsubscribe = usersWorkspaceStore.subscribe((list) => list && resolve()) + }) + unsubscribe() +} diff --git a/frontend/src/lib/remoteDeploy.ts b/frontend/src/lib/remoteDeploy.ts index ad887504fb..dbac372f94 100644 --- a/frontend/src/lib/remoteDeploy.ts +++ b/frontend/src/lib/remoteDeploy.ts @@ -1,4 +1,3 @@ -import { base } from '$lib/base' import type { RemoteDeployConnection, RemoteDeployTarget } from '$lib/gen' import { randomSecret } from '$lib/utils/uuid' @@ -67,7 +66,7 @@ export function remoteDeployAuthorizeUrl( localStorage.setItem(PENDING_PREFIX + state, JSON.stringify(pending)) const params = new URLSearchParams({ workspace: target.workspace_id, - callback: `${window.location.origin}${base}${CALLBACK_PATH}`, + callback: `${window.location.origin}${CALLBACK_PATH}`, state }) return `${target.base_url}/user/remote_deploy_authorize?${params}` diff --git a/frontend/src/lib/scripts.ts b/frontend/src/lib/scripts.ts index 2d905d690a..f9f1fa775e 100644 --- a/frontend/src/lib/scripts.ts +++ b/frontend/src/lib/scripts.ts @@ -1,5 +1,4 @@ import { get } from 'svelte/store' -import { base } from '$lib/base' import type { Schema, SupportedLanguage } from './common' import { FlowService, type Script, ScriptService, ScheduleService } from './gen' import { hubBaseUrlStore, workspaceStore } from './stores' @@ -136,7 +135,7 @@ export function scriptPathToHref(path: string, hubBaseUrl: string): string { if (path.startsWith('hub/')) { return hubBaseUrl + '/from_version/' + path.substring(4) } else { - return `${base}/scripts/get/${path}?workspace=${get(workspaceStore)}` + return `/scripts/get/${path}?workspace=${get(workspaceStore)}` } } @@ -146,7 +145,7 @@ export function flowPathToHref(path: string, hubBaseUrl: string = get(hubBaseUrl return hubFlowId ? `${hubBaseUrl}/flows/${hubFlowId}` : hubBaseUrl } - return `${base}/flows/get/${path}?workspace=${get(workspaceStore)}` + return `/flows/get/${path}?workspace=${get(workspaceStore)}` } const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string][] = [ @@ -175,8 +174,8 @@ const scriptLanguagesArray: [SupportedLanguage | 'docker' | 'bunnative', string] ['ruby', 'Ruby'], ['rlang', 'R'], ['dbt', 'dbt'], - // This array's order is the picker order. Deno is de-emphasized ahead of - // deprecation, so it stays last. + // Not offered for new scripts (see `processLangs`); listed for the label of + // existing ones. ['deno', 'Deno'] // for related places search: ADD_NEW_LANG ] @@ -197,6 +196,12 @@ export function processInlineLangs(selected: string | undefined, langs: string[] } export function processLangs(selected: string | undefined, langs: string[]): string[] { + // Only a script already written in deno keeps the option, whether or not the + // workspace's saved order still carries it. + langs = langs.filter((lang) => lang !== 'deno') + if (selected === 'deno') { + langs = [...langs, 'deno'] + } if (selected === 'nativets') { return langs } else { diff --git a/frontend/src/lib/utils/editInFork.ts b/frontend/src/lib/utils/editInFork.ts index 67eebcf7eb..d43c3644c9 100644 --- a/frontend/src/lib/utils/editInFork.ts +++ b/frontend/src/lib/utils/editInFork.ts @@ -1,4 +1,3 @@ -import { base } from '$lib/base' import { get } from 'svelte/store' import { userStore, @@ -75,13 +74,13 @@ export function canCreateFork(user: UserExt | undefined): boolean { function editPathFor(itemType: ItemType, itemPath: string): string { switch (itemType) { case 'script': - return `${base}/scripts/edit/${itemPath}` + return `/scripts/edit/${itemPath}` case 'flow': - return `${base}/flows/edit/${itemPath}` + return `/flows/edit/${itemPath}` case 'app': - return `${base}/apps/edit/${itemPath}` + return `/apps/edit/${itemPath}` case 'raw_app': - return `${base}/apps_raw/edit/${itemPath}` + return `/apps_raw/edit/${itemPath}` } } @@ -101,7 +100,7 @@ export function buildForkEditUrl( * workspace the page loads in, so a caller acting on another one names it. */ export function forkWorkspaceUrl(itemType: ItemType, itemPath: string, workspace?: string): string { const from = workspace ? `&workspace=${encodeURIComponent(workspace)}` : '' - return `${base}/user/fork_workspace?rd=${encodeURIComponent(editPathFor(itemType, itemPath))}${from}` + return `/user/fork_workspace?rd=${encodeURIComponent(editPathFor(itemType, itemPath))}${from}` } /** The item's editor in the dev workspace — the target `buildForkEditUrl` produces when a dev exists. */ diff --git a/frontend/src/lib/utils_deployable.ts b/frontend/src/lib/utils_deployable.ts index 63ea88adcb..e3f7fa83ed 100644 --- a/frontend/src/lib/utils_deployable.ts +++ b/frontend/src/lib/utils_deployable.ts @@ -16,7 +16,7 @@ import { type WorkspaceDeployUISettings } from './gen' import type { TriggerKind } from './components/triggers' -import { base } from './base' +import { apiBaseUrl } from './apiBaseUrl.svelte' type DeployUIType = 'script' | 'flow' | 'app' | 'resource' | 'variable' | 'secret' | 'trigger' @@ -250,9 +250,7 @@ export async function getTriggersDeployData( ...gcpTrigger, delivery_config: gcpTrigger.delivery_config ?? undefined, base_endpoint: - gcpTrigger.delivery_type === 'push' - ? (targetBaseUrl ?? `${window.location.origin}${base}`) - : undefined, + gcpTrigger.delivery_type === 'push' ? (targetBaseUrl ?? apiBaseUrl()) : undefined, permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs } @@ -320,7 +318,7 @@ export async function getTriggersDeployData( base_endpoint: azureTrigger.azure_mode === 'namespace_pull' ? undefined - : (targetBaseUrl ?? `${window.location.origin}${base}`), + : (targetBaseUrl ?? apiBaseUrl()), permissioned_as: onBehalfOf, preserve_permissioned_as: preservePermissionedAs }, diff --git a/frontend/src/lib/utils_workspace_deploy.ts b/frontend/src/lib/utils_workspace_deploy.ts index 280d0daa00..fa50805e3a 100644 --- a/frontend/src/lib/utils_workspace_deploy.ts +++ b/frontend/src/lib/utils_workspace_deploy.ts @@ -314,7 +314,7 @@ function makeProvider( existsTriggerByKind: (kind, p) => triggerServiceFor(kind).exists(p), getTriggerForDeploy: async (kind, p) => { // Reuses the existing per-kind transform map (e.g. GCP wipes - // subscription_id and computes base_endpoint from window.location). + // subscription_id and computes base_endpoint for this instance). // Operational-state strip is applied by the shared `deployItem` // after this returns. const { data } = await getTriggersDeployData( diff --git a/frontend/src/lib/workspaceCreation.ts b/frontend/src/lib/workspaceCreation.ts index 34c1644462..0a31d66a96 100644 --- a/frontend/src/lib/workspaceCreation.ts +++ b/frontend/src/lib/workspaceCreation.ts @@ -13,7 +13,6 @@ import { SettingService, UserService, WorkspaceService } from '$lib/gen' import { usersWorkspaceStore } from '$lib/stores' import { switchWorkspace } from '$lib/storeUtils' import { isCloudHosted } from '$lib/cloud' -import { base } from '$lib/base' import { WORKSPACE_NAME_MAX_LENGTH } from '$lib/utils/workspaceId' /** @@ -30,7 +29,7 @@ import { WORKSPACE_NAME_MAX_LENGTH } from '$lib/utils/workspaceId' export async function canCreateWorkspace(isSuperadmin: boolean): Promise { if (isSuperadmin || isCloudHosted()) return true try { - const r = await fetch(base + '/api/workspaces/create_workspace_require_superadmin') + const r = await fetch('/api/workspaces/create_workspace_require_superadmin') if ((await r.text()) != 'true') return true return !!(await UserService.globalWhoami()).super_admin } catch { diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 2403299e71..5edf490220 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -102,18 +102,17 @@ import InstanceUiSync from '$lib/components/InstanceUiSync.svelte' import { onDestroy, setContext, untrack } from 'svelte' import { cubicOut } from 'svelte/easing' - import { base } from '$app/paths' import { Menubar, Tooltip } from '$lib/components/meltComponents' import { aiChatManager } from '$lib/components/copilot/chat/AIChatManager.svelte' import AiChatLayout from '$lib/components/copilot/chat/AiChatLayout.svelte' import SessionPicker from '$lib/components/sessions/SessionPicker.svelte' import SessionModeSwitch from '$lib/components/sessions/SessionModeSwitch.svelte' - import { isGlobalAiEnabled } from '$lib/components/copilot/chat/global/gate' + import { isGlobalAiEnabled, sessionsAllowedFor } from '$lib/components/copilot/chat/global/gate' import { copilotInfo } from '$lib/aiStore' import { isPageItemListPath, parsePreviewItemRoute, - stripBase + barePath } from '$lib/components/sessions/previewPaths' import { rememberNavRoute } from '$lib/components/sessions/sessionSwitch.svelte' import { sessionState } from '$lib/components/sessions/sessionState.svelte' @@ -300,11 +299,10 @@ // Session mode is route-derived: the rail shows the sessions sidebar on the // /sessions page and the workspace navigation everywhere else. The switch // (SessionModeSwitch) just navigates in and out of that route. - // The route decides, except for an operator: /sessions refuses them, so handing their - // sidebar over to a session list would offer what the page behind it is declining. - let sessionMode = $derived( - page.url.pathname.startsWith(base + '/sessions') && !$userStore?.operator - ) + // An operator without the opt-in (`gate.ts`) is refused by /sessions, so their sidebar is not + // handed over to a session list the page behind it declines, and they keep the docked chat. + const sessionsAllowed = $derived(sessionsAllowedFor($userStore?.operator)) + let sessionMode = $derived(page.url.pathname.startsWith('/sessions') && sessionsAllowed) // Session mode points the bottom settings entry at the open session's own // workspace. An unsent draft hasn't committed one yet, so it falls back to // the family root. @@ -326,12 +324,10 @@ // A workspace that hid the assistant (`ai_config.copilot_disabled`) loses both entry // points: the Workspace ⇄ Sessions switch and the legacy Ask-AI button. const sessionsSwitchShown = $derived( - globalAiEnabled && !$copilotInfo.workspaceDisabled && !$userStore?.operator + globalAiEnabled && sessionsAllowed && !$copilotInfo.workspaceDisabled ) - // The legacy pane's entry point. An operator keeps that pane whatever the beta gate says - // (`disableAi` above), so the sidebar must still offer them a way into it. const askAiShown = $derived( - (!globalAiEnabled || $userStore?.operator) && !$copilotInfo.workspaceDisabled + (!globalAiEnabled || !sessionsAllowed) && !$copilotInfo.workspaceDisabled ) if (page.status == 404) { @@ -471,7 +467,7 @@ } // A list page mounts in process too, where its rows open as tabs of their own. const listUrl = navigation.to?.url - if (listUrl && isPageItemListPath(stripBase(listUrl.pathname))) { + if (listUrl && isPageItemListPath(barePath(listUrl.pathname))) { navigation.cancel() const u = new URL(listUrl.href) u.searchParams.delete('nomenubar') @@ -722,7 +718,7 @@ // Remember the last navigation-mode route so exiting session mode returns // the user where they were rather than to the home page. const to = n.to?.url - if (to && !to.pathname.startsWith(base + '/sessions')) { + if (to && !to.pathname.startsWith('/sessions')) { rememberNavRoute(to.pathname + to.search) } }) @@ -737,7 +733,7 @@ } } - let devOnly = $derived(page.url.pathname.startsWith(base + '/scripts/dev')) + let devOnly = $derived(page.url.pathname.startsWith('/scripts/dev')) async function loadDefaultScripts(workspace: string, user: UserExt | undefined) { if (!user?.operator) { @@ -891,7 +887,7 @@ $effect(() => { const ws = $workspaceStore const ready = sessionState.hydrated && $usersWorkspaceStore !== undefined - if (globalAiEnabled && ready && ws && !$userStore?.operator) { + if (globalAiEnabled && sessionsAllowed && ready && ws) { untrack(() => restoreSessionBackups(ws)) } }) @@ -1054,7 +1050,7 @@
diff --git a/frontend/src/routes/(root)/(logged)/agents/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/agents/get/[...path]/+page.svelte index 30eb5cb32c..ceb37c0a10 100644 --- a/frontend/src/routes/(root)/(logged)/agents/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/agents/get/[...path]/+page.svelte @@ -2,7 +2,6 @@ import { page } from '$app/state' import { FlaskConical, FormInput, MessageSquare, Pen } from 'lucide-svelte' import { resource } from 'runed' - import { base } from '$lib/base' import { goto } from '$lib/navigation' import { copilotInfo } from '$lib/aiStore' import AgentEvalsModal from '$lib/components/flows/content/AgentEvalsModal.svelte' @@ -15,10 +14,12 @@ import AgentEditorHost from '$lib/components/flows/content/AgentEditorHost.svelte' import AgentConfigModal from '$lib/components/flows/content/AgentConfigModal.svelte' import RunForm from '$lib/components/RunForm.svelte' + import { Alert } from '$lib/components/common' import { keepsManagedMemory } from '$lib/components/flows/agentFormFields' import { agentMenuItems, deleteAgent } from '$lib/components/flows/agentActions' import { getDeployUiSettings } from '$lib/components/home/deploy_ui' import { userStore, workspaceStore } from '$lib/stores' + import { sendUserToast } from '$lib/toast' /** * The deployed agent, as a flow's or a script's page shows theirs: a way to run it, a chat when @@ -41,9 +42,24 @@ let deploymentDrawer: DeployWorkspaceDrawer | undefined = $state(undefined) let deleteOpen = $state(false) let evalsModal: AgentEvalsModal | undefined = $state(undefined) + let runForm: RunForm | undefined = $state(undefined) + let isValid = $state(true) + + // Cmd/Ctrl+Enter runs the form, as on a script's or a flow's page. The form stays mounted, hidden, + // while the chat shows, and the chat's composer owns the key then. + function onKeyDown(event: KeyboardEvent) { + if (event.defaultPrevented || event.key !== 'Enter' || !(event.ctrlKey || event.metaKey)) return + if (!runForm || testPane?.mode === 'chat') return + if (isValid) { + event.preventDefault() + runForm.run() + } else { + sendUserToast('Please fix errors before running', true) + } + } async function remove() { - if (ws && (await deleteAgent(ws, path))) await goto(`${base}/?kind=agent`) + if (ws && (await deleteAgent(ws, path))) await goto('/?kind=agent') } // The workspace's, so a path change never waits on it and a menu never outlives its path. @@ -88,6 +104,8 @@ Every flow that links {path} will fail at its agent step once it is deleted. + +
+ {#if failedTopLevelStep && failedTopLevelStep !== AGENT_STEP_ID} at step {failedTopLevelStep} {#if $enterpriseLicense && canRestart(failedRestart)} · diff --git a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte index 97ed4b95b6..114c6554aa 100644 --- a/frontend/src/routes/(root)/(logged)/sessions/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/sessions/+page.svelte @@ -54,12 +54,12 @@ import { markSessionRecovered } from '$lib/components/sessions/sessionRecoveryNotice.svelte' import { isGlobalAiEnabled, - clearSessionsBetaOptOut + clearSessionsBetaOptOut, + sessionsAllowedFor } from '$lib/components/copilot/chat/global/gate' import { setToolCompletionListener } from '$lib/components/copilot/chat/shared' import { registerToolDisplayActionHandler } from '$lib/components/copilot/chat/createdResourceActions.svelte' import { previewTargetForSessionTarget } from '$lib/components/sessions/sessionPreviewTabs.svelte' - import { base } from '$lib/base' import { artifactKey, itemDisplayName, @@ -80,7 +80,7 @@ tabsToReload, viewerTabsToReload } from '$lib/components/sessions/previewReload' - import { isPageItemListPath, stripBase } from '$lib/components/sessions/previewPaths' + import { isPageItemListPath, barePath } from '$lib/components/sessions/previewPaths' import { leafKeyFor, loadKind, @@ -205,7 +205,7 @@ // load. `recovering` also guards re-entry: recovery mutates the session list // this effect tracks, while the URL that would stop it only updates on `goto`. $effect(() => { - if (embedded || !globalEnabled || $userStore?.operator) return + if (embedded || !globalEnabled || !sessionsAllowedFor($userStore?.operator)) return if (!sessionState.hydrated || recovering) return // A deliberate delete removes the open session ahead of its own navigation. // Claiming that gap would take over the URL and tell the user the session @@ -593,10 +593,10 @@ // Page path shown after the workspace breadcrumb — the active tab's observed // location, so the breadcrumb tracks where the user browses inside the tab. - const displayPath = $derived(owner?.activeTab?.loc ?? owner?.activeTab?.url ?? `${base}/`) + const displayPath = $derived(owner?.activeTab?.loc ?? owner?.activeTab?.url ?? '/') const activeArtifact = $derived(owner?.activeTab ? parseArtifactRoute(owner.activeTab.url) : null) const activeWorkspaceHref = $derived( - owner?.activeTab ? workspacePageHref(owner.activeTab.loc || owner.activeTab.url) : `${base}/` + owner?.activeTab ? workspacePageHref(owner.activeTab.loc || owner.activeTab.url) : '/' ) // The active session's artifacts, surfaced as an "Artifacts" branch in the // preview pickers. @@ -658,7 +658,7 @@ const tabHosts: Record = {} let reloadHandle: ReturnType | undefined - // Per workspace a chat round touched since the last flush: base-stripped list-page paths + // Per workspace a chat round touched since the last flush: list-page paths // (e.g. `/schedules`, see toolReloadEffect), the page items its tools named, as tab urls, // and the item paths whose deployed version changed. By workspace because a path names an // item only within one: a write in one fork must not remount the same path's editor in a @@ -889,7 +889,7 @@ // Leaving the preview for the standalone editor — carry the session // workspace so the app opens in the fork the session acts on, not the // navigation workspace. - goto(withWorkspaceParam(`${base}/apps/edit/${item.path}`, previewWorkspace)) + goto(withWorkspaceParam(`/apps/edit/${item.path}`, previewWorkspace)) return } owner?.navigate({ type: 'item', item }) @@ -928,7 +928,7 @@ // A preview frame navigating to a list page: re-point the active tab to the list // mounted in process, whose rows open as tabs of their own. if (d.type === 'wm.session.openList') { - if (typeof d.href !== 'string' || !isPageItemListPath(stripBase(d.href))) return + if (typeof d.href !== 'string' || !isPageItemListPath(barePath(d.href))) return owner?.navigate({ type: 'page', href: d.href, label: previewLocationLabel(d.href) }) return } @@ -1021,9 +1021,9 @@ }}>Open sessions
- {:else if $userStore?.operator} - + {:else if !sessionsAllowedFor($userStore?.operator)} +

AI Sessions are not available for operators

Use the Ask AI chat instead.

@@ -1033,7 +1033,7 @@ try { localStorage.setItem('ai-chat-open', 'true') } catch {} - window.location.href = `${base}/` + window.location.href = '/' }} > Open Ask AI chat @@ -1045,7 +1045,7 @@

AI Sessions are deactivated

You switched back to the legacy chat. Activate AI Sessions (beta) to open this page.

-
diff --git a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte index d4ac558cb8..212696782b 100644 --- a/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte +++ b/frontend/src/routes/(root)/(logged)/svix/create-webhook/+page@(root).svelte @@ -4,6 +4,8 @@ import ScriptPicker from '$lib/components/ScriptPicker.svelte' import { Button } from '$lib/components/common' import { page } from '$app/stores' + import { apiBaseUrl } from '$lib/apiBaseUrl.svelte' + import InstanceUiSync from '$lib/components/InstanceUiSync.svelte' import { userStore, workspaceStore } from '$lib/stores' import { sendUserToast } from '$lib/toast' import UserSettings from '$lib/components/UserSettings.svelte' @@ -26,11 +28,11 @@ let webhook: string switch (kind) { case 'script': - webhook = `${$page.url.origin}/api/w/${$workspaceStore}/jobs/run/p/${path}` + webhook = `${apiBaseUrl()}/api/w/${$workspaceStore}/jobs/run/p/${path}` redirectSuccess(webhook, token) break case 'flow': - webhook = `${$page.url.origin}/api/w/${$workspaceStore}/jobs/run/f/${path}` + webhook = `${apiBaseUrl()}/api/w/${$workspaceStore}/jobs/run/f/${path}` redirectSuccess(webhook, token) break default: @@ -72,6 +74,8 @@ }) + + { diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/accept_invite/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/accept_invite/+page.svelte index e14a2113cb..6b731a4833 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/accept_invite/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/accept_invite/+page.svelte @@ -2,7 +2,6 @@ import { run } from 'svelte/legacy'; import { goto } from '$lib/navigation' - import { base } from '$lib/base' import { SettingService, UserService, WorkspaceService } from '$lib/gen' import { validateUsername } from '$lib/utils' @@ -84,7 +83,7 @@ {/if}
- -

- Fetches the latest resource types directly from the Windmill Hub (requires internet - access). -

- {#if hubSyncStatus === 'success'} - - {hubSyncMessage} - - {:else if hubSyncStatus === 'error'} - - {hubSyncMessage} - - {/if}

- The daily schedule synchronizes resource types from the Hub every day at midnight UTC. + The server updates them from the Hub once a day, overwriting local edits to the types the + Hub also defines. You can change this later in the instance settings.

diff --git a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte index 240eee8473..8bcd94ce24 100644 --- a/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/user/(user)/login/+page.svelte @@ -1,5 +1,4 @@ @@ -1464,7 +1469,11 @@ - +
@@ -1544,7 +1553,7 @@ @@ -1582,7 +1591,7 @@ @@ -1634,9 +1643,9 @@ } }} onSelect={editSlackCommand} - connectHref="{base}/api/oauth/connect_slack" - createScriptHref="{base}/scripts/add?hub=hub%2F28071%2Fslack%2Fexample_of_responding_to_a_slack_command_slack" - createFlowHref="{base}/flows/add?hub=28" + connectHref="/api/oauth/connect_slack" + createScriptHref="/scripts/add?hub=hub%2F28071%2Fslack%2Fexample_of_responding_to_a_slack_command_slack" + createFlowHref="/flows/add?hub=28" documentationLink="https://www.windmill.dev/docs/integrations/slack" onLoadSettings={loadSettings} display_name={slack_team_name} @@ -1721,7 +1730,7 @@ Slack API . Set the redirect URI to:{' '} - {window.location.origin}{base}/oauth/callback_slack + {window.location.origin}/oauth/callback_slack
@@ -1771,8 +1780,8 @@ }} onSelect={editTeamsCommand} connectHref={undefined} - createScriptHref="{base}/scripts/add?hub=hub%2F11591%2Fteams%2FExample%20of%20responding%20to%20a%20Microsoft%20Teams%20command" - createFlowHref="{base}/flows/add?hub=58" + createScriptHref="/scripts/add?hub=hub%2F11591%2Fteams%2FExample%20of%20responding%20to%20a%20Microsoft%20Teams%20command" + createFlowHref="/flows/add?hub=58" documentationLink="https://www.windmill.dev/docs/integrations/teams" onLoadSettings={loadSettings} display_name={teams_team_name} @@ -1825,7 +1834,7 @@ {:else}