diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index d0bfd53ae8..8cac56c35b 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,10 +1,8 @@ env: REGISTRY: ghcr.io - IMAGE_NAME: - ${{ github.event_name != 'pull_request' && github.repository || + IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.repository || 'windmill-labs/windmill-test' }} - DEV_SHA: - ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}', + DEV_SHA: ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}', github.event.number) }} name: Build windmill:main diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e06756175..dcf9f6a2a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,51 @@ # Changelog +## [1.398.1](https://github.com/windmill-labs/windmill/compare/v1.398.0...v1.398.1) (2024-09-23) + + +### Bug Fixes + +* time handling in bun and deno ([#4423](https://github.com/windmill-labs/windmill/issues/4423)) ([61f27ac](https://github.com/windmill-labs/windmill/commit/61f27acbd5b477f0e2453d4b02d406a7aef76009)) + +## [1.398.0](https://github.com/windmill-labs/windmill/compare/v1.397.4...v1.398.0) (2024-09-23) + + +### Features + +* **frontend:** add http routing templates ([#4421](https://github.com/windmill-labs/windmill/issues/4421)) ([e99e7b2](https://github.com/windmill-labs/windmill/commit/e99e7b2b0bb1bc63d1253d147b80bac06eaff103)) +* http routing ([#4339](https://github.com/windmill-labs/windmill/issues/4339)) ([304dac3](https://github.com/windmill-labs/windmill/commit/304dac34475b8871621c54a768275fae1e9f845e)) + + +### Bug Fixes + +* allow no body in job requests ([#4413](https://github.com/windmill-labs/windmill/issues/4413)) ([70fa78c](https://github.com/windmill-labs/windmill/commit/70fa78c10d8afb3e06be84e3449fdfdeb2673765)) +* **frontend:** Fix delete branch one index ([#4418](https://github.com/windmill-labs/windmill/issues/4418)) ([30017cc](https://github.com/windmill-labs/windmill/commit/30017cc2b1401ea439e3fbec5d7764b206100a38)) +* migrate smtp instance settings to global settings ([#4416](https://github.com/windmill-labs/windmill/issues/4416)) ([50a6f78](https://github.com/windmill-labs/windmill/commit/50a6f789fa8a03d70407b0429da049cc0740b6fa)) +* no failed renewal alert if trial ([#4414](https://github.com/windmill-labs/windmill/issues/4414)) ([5be7be0](https://github.com/windmill-labs/windmill/commit/5be7be03a6df34a9fae96ed7f844d693f7cfce3e)) +* update pip to 24.2 and python 3.11.8->3.11.10 ([a17195d](https://github.com/windmill-labs/windmill/commit/a17195d88ff52b72f0c2cdd95c6218df8ef6d7cf)) +* various improvements for ansible ([#4419](https://github.com/windmill-labs/windmill/issues/4419)) ([a500994](https://github.com/windmill-labs/windmill/commit/a500994cc2cd9cc6cbad722059cebd0bf74a8850)) + +## [1.397.4](https://github.com/windmill-labs/windmill/compare/v1.397.3...v1.397.4) (2024-09-20) + + +### Bug Fixes + +* **cli:** make CLI not require latest version of windmill ([794f87a](https://github.com/windmill-labs/windmill/commit/794f87aa0d33a0b782fcb2726754fdc51bc1d73d)) + +## [1.397.3](https://github.com/windmill-labs/windmill/compare/v1.397.2...v1.397.3) (2024-09-20) + + +### Bug Fixes + +* **cli:** make CLI compatible with Node 18 ([8212532](https://github.com/windmill-labs/windmill/commit/8212532b29fbcf971cb320b8cfc84f3a8d8ad795)) + +## [1.397.2](https://github.com/windmill-labs/windmill/compare/v1.397.1...v1.397.2) (2024-09-20) + + +### Bug Fixes + +* **bun:** never pre-bundle BASE_URL env variable ([69b8754](https://github.com/windmill-labs/windmill/commit/69b8754aef32682bf812be783f79b81cd8526c7a)) + ## [1.397.1](https://github.com/windmill-labs/windmill/compare/v1.397.0...v1.397.1) (2024-09-20) diff --git a/Dockerfile b/Dockerfile index a1c77f7e13..3bc5088aed 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG DEBIAN_IMAGE=debian:bookworm-slim ARG RUST_IMAGE=rust:1.80-slim-bookworm -ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm +ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm FROM ${RUST_IMAGE} AS rust_base @@ -95,6 +95,8 @@ ARG WITH_KUBECTL=true ARG WITH_HELM=true ARG WITH_GIT=true +RUN pip install --upgrade pip==24.2 + RUN apt-get update \ && apt-get install -y ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1 software-properties-common \ && apt-get clean \ diff --git a/backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json b/backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json new file mode 100644 index 0000000000..a27bc6365a --- /dev/null +++ b/backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json @@ -0,0 +1,89 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_async", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "requires_auth", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "http_method: _", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441" +} diff --git a/backend/.sqlx/query-0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90.json b/backend/.sqlx/query-0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90.json deleted file mode 100644 index 4f6b896ad4..0000000000 --- a/backend/.sqlx/query-0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90" -} diff --git a/backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json b/backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json new file mode 100644 index 0000000000..00361e6e95 --- /dev/null +++ b/backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63" +} diff --git a/backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json b/backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json new file mode 100644 index 0000000000..4000bbb962 --- /dev/null +++ b/backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), ARRAY['step'], $2)\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f" +} diff --git a/backend/.sqlx/query-872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e.json b/backend/.sqlx/query-3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d.json similarity index 57% rename from backend/.sqlx/query-872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e.json rename to backend/.sqlx/query-3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d.json index cdf6a3f4fa..1d91cd4031 100644 --- a/backend/.sqlx/query-872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e.json +++ b/backend/.sqlx/query-3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return \n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", + "query": "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor \n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", "describe": { "columns": [ { @@ -15,8 +15,8 @@ }, { "ordinal": 2, - "name": "early_return", - "type_info": "Text" + "name": "has_preprocessor", + "type_info": "Bool" } ], "parameters": { @@ -31,5 +31,5 @@ null ] }, - "hash": "872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e" + "hash": "3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d" } diff --git a/backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json b/backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json new file mode 100644 index 0000000000..53a59847e7 --- /dev/null +++ b/backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE completed_job SET args = '{\"reason\":\"PREPROCESSOR_ARGS_ARE_DISCARDED\"}'::jsonb WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f" +} diff --git a/backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json b/backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json new file mode 100644 index 0000000000..b66edc6d73 --- /dev/null +++ b/backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE http_trigger \n SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() \n WHERE workspace_id = $11 AND path = $12", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Varchar", + "Varchar", + "Bool", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea" +} diff --git a/backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json b/backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json new file mode 100644 index 0000000000..d954f2c26c --- /dev/null +++ b/backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json @@ -0,0 +1,108 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth\n FROM http_trigger\n WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "route_path_key", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "http_method: _", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + }, + { + "ordinal": 7, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 10, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 11, + "name": "is_async", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "requires_auth", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491" +} diff --git a/backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json b/backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json new file mode 100644 index 0000000000..ca0a922067 --- /dev/null +++ b/backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() \n WHERE workspace_id = $9 AND path = $10", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Varchar", + "Varchar", + "Bool", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0" +} diff --git a/backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json b/backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json similarity index 89% rename from backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json rename to backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json index ebd2e2535d..6bc0fb8a10 100644 --- a/backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json +++ b/backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2", + "query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where hash = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -82,6 +82,11 @@ "ordinal": 10, "name": "timeout", "type_info": "Int4" + }, + { + "ordinal": 11, + "name": "has_preprocessor", + "type_info": "Bool" } ], "parameters": { @@ -101,8 +106,9 @@ true, true, true, + true, true ] }, - "hash": "8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c" + "hash": "64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac" } diff --git a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json index 1fa370e682..75b8108281 100644 --- a/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json +++ b/backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json @@ -5,7 +5,7 @@ "columns": [ { "ordinal": 0, - "name": "bool", + "name": "?column?", "type_info": "Bool" } ], diff --git a/backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json b/backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json similarity index 52% rename from backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json rename to backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json index f05a1b36ac..4e0bc23b69 100644 --- a/backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json +++ b/backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT config FROM config WHERE name = 'server'", + "query": "SELECT value FROM global_settings WHERE name = 'smtp_settings'", "describe": { "columns": [ { "ordinal": 0, - "name": "config", + "name": "value", "type_info": "Jsonb" } ], @@ -13,8 +13,8 @@ "Left": [] }, "nullable": [ - true + false ] }, - "hash": "8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3" + "hash": "83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b" } diff --git a/backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json b/backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json new file mode 100644 index 0000000000..ec6143fe39 --- /dev/null +++ b/backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd" +} diff --git a/backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json b/backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json new file mode 100644 index 0000000000..f5c36ed27f --- /dev/null +++ b/backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n WHERE id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368" +} diff --git a/backend/.sqlx/query-98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba.json b/backend/.sqlx/query-98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba.json new file mode 100644 index 0000000000..05cbaffc69 --- /dev/null +++ b/backend/.sqlx/query-98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor\n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "dedicated_worker", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "early_return", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "has_preprocessor", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + true, + null, + null + ] + }, + "hash": "98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba" +} diff --git a/backend/.sqlx/query-9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb.json b/backend/.sqlx/query-9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb.json new file mode 100644 index 0000000000..a0c5bd7642 --- /dev/null +++ b/backend/.sqlx/query-9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE path = $1 AND workspace_id = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb" +} diff --git a/backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json b/backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json new file mode 100644 index 0000000000..5de165462e --- /dev/null +++ b/backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb" +} diff --git a/backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json b/backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json new file mode 100644 index 0000000000..ae30d3bd98 --- /dev/null +++ b/backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef" +} diff --git a/backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json b/backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json new file mode 100644 index 0000000000..00453be45b --- /dev/null +++ b/backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa" +} diff --git a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json index 54e94cfb8f..99269c9851 100644 --- a/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json +++ b/backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json @@ -18,8 +18,8 @@ "Left": [] }, "nullable": [ - false, - true + true, + false ] }, "hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76" diff --git a/backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json b/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json similarity index 85% rename from backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json rename to backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json index 72ab4ffe26..71aaf4ba90 100644 --- a/backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json +++ b/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31)", "describe": { "columns": [], "parameters": { @@ -72,10 +72,11 @@ "Varchar", "Bool", "Bool", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [] }, - "hash": "8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb" + "hash": "b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3" } diff --git a/backend/.sqlx/query-b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f.json b/backend/.sqlx/query-b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f.json new file mode 100644 index 0000000000..a730d4b588 --- /dev/null +++ b/backend/.sqlx/query-b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1 AND path = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f" +} diff --git a/backend/.sqlx/query-b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9.json b/backend/.sqlx/query-b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9.json deleted file mode 100644 index 948e713ef6..0000000000 --- a/backend/.sqlx/query-b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n WHERE id = $4", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9" -} diff --git a/backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json b/backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json new file mode 100644 index 0000000000..7e9e9b84aa --- /dev/null +++ b/backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772" +} diff --git a/backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json b/backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json new file mode 100644 index 0000000000..df94d893a6 --- /dev/null +++ b/backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Bool", + "Bool", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8" +} diff --git a/backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json b/backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json new file mode 100644 index 0000000000..03b0c561e1 --- /dev/null +++ b/backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188" +} diff --git a/backend/.sqlx/query-f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8.json b/backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json similarity index 74% rename from backend/.sqlx/query-f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8.json rename to backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json index a0a4a3770c..39c96b8989 100644 --- a/backend/.sqlx/query-f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8.json +++ b/backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1", + "query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1", "describe": { "columns": [ { @@ -24,5 +24,5 @@ null ] }, - "hash": "f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8" + "hash": "f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929" } diff --git a/backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json b/backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json similarity index 83% rename from backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json rename to backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json index 38c8521741..5e4f522147 100644 --- a/backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json +++ b/backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", + "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", "describe": { "columns": [ { @@ -82,6 +82,11 @@ "ordinal": 10, "name": "timeout", "type_info": "Int4" + }, + { + "ordinal": 11, + "name": "has_preprocessor", + "type_info": "Bool" } ], "parameters": { @@ -101,8 +106,9 @@ true, true, true, + true, true ] }, - "hash": "e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99" + "hash": "fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a" } diff --git a/backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json b/backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json new file mode 100644 index 0000000000..65cb5c6633 --- /dev/null +++ b/backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json @@ -0,0 +1,87 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_async", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "requires_auth", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "http_method: _", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 48eae53d77..dfa5b71823 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1056,9 +1056,9 @@ dependencies = [ [[package]] name = "axum" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec" dependencies = [ "async-trait", "axum-core", @@ -1083,7 +1083,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper 1.0.1", "tokio", - "tower 0.4.13", + "tower 0.5.1", "tower-layer", "tower-service", "tracing", @@ -1091,9 +1091,9 @@ dependencies = [ [[package]] name = "axum-core" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00" dependencies = [ "async-trait", "bytes", @@ -1104,7 +1104,7 @@ dependencies = [ "mime", "pin-project-lite", "rustversion", - "sync_wrapper 0.1.2", + "sync_wrapper 1.0.1", "tower-layer", "tower-service", "tracing", @@ -1735,9 +1735,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac" +checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3" dependencies = [ "clap_builder", "clap_derive", @@ -1745,9 +1745,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.17" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73" +checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b" dependencies = [ "anstream", "anstyle", @@ -1757,9 +1757,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.13" +version = "4.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0" +checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab" dependencies = [ "heck 0.5.0", "proc-macro2", @@ -5442,7 +5442,7 @@ dependencies = [ "md-5 0.10.6", "parking_lot", "percent-encoding", - "quick-xml 0.36.1", + "quick-xml 0.36.2", "rand 0.8.5", "reqwest 0.12.7", "ring 0.17.8", @@ -5991,9 +5991,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "polling" @@ -6012,9 +6012,9 @@ dependencies = [ [[package]] name = "portable-atomic" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265" +checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce" [[package]] name = "postgres-native-tls" @@ -6374,9 +6374,9 @@ dependencies = [ [[package]] name = "quick-xml" -version = "0.36.1" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" +checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe" dependencies = [ "memchr", "serde", @@ -7415,9 +7415,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.11.1" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" +checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6" dependencies = [ "core-foundation-sys", "libc", @@ -7755,9 +7755,9 @@ dependencies = [ [[package]] name = "simdutf8" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" [[package]] name = "simple_asn1" @@ -9061,18 +9061,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" +checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" +checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3" dependencies = [ "proc-macro2", "quote", @@ -9497,7 +9497,6 @@ dependencies = [ "tokio", "tower-layer", "tower-service", - "tracing", ] [[package]] @@ -9506,8 +9505,14 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 0.1.2", + "tokio", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -10421,7 +10426,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "axum", @@ -10463,7 +10468,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "argon2", @@ -10497,6 +10502,7 @@ dependencies = [ "lazy_static", "magic-crypt", "mail-parser", + "matchit", "mime_guess", "native-tls", "object_store", @@ -10547,7 +10553,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.397.1" +version = "1.398.1" dependencies = [ "base64 0.21.7", "chrono", @@ -10565,7 +10571,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.397.1" +version = "1.398.1" dependencies = [ "chrono", "serde", @@ -10578,7 +10584,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "async-stream", @@ -10622,7 +10628,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.397.1" +version = "1.398.1" dependencies = [ "regex", "rsmq_async", @@ -10637,7 +10643,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "bytes", @@ -10658,7 +10664,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.397.1" +version = "1.398.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -10667,7 +10673,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "lazy_static", @@ -10679,7 +10685,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "gosyn", @@ -10691,7 +10697,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "lazy_static", @@ -10703,7 +10709,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10714,7 +10720,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10725,7 +10731,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "async-recursion", @@ -10743,7 +10749,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10760,7 +10766,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "lazy_static", @@ -10772,7 +10778,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "lazy_static", @@ -10790,7 +10796,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10811,7 +10817,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "serde_json", @@ -10821,7 +10827,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "async-recursion", @@ -10854,7 +10860,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.397.1" +version = "1.398.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10864,7 +10870,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.397.1" +version = "1.398.1" dependencies = [ "anyhow", "async-recursion", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index c77b140846..2f04b941b1 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.397.1" +version = "1.398.1" authors.workspace = true edition.workspace = true @@ -27,7 +27,7 @@ members = [ ] [workspace.package] -version = "1.397.1" +version = "1.398.1" authors = ["Ruben Fiszel "] edition = "2021" @@ -252,6 +252,7 @@ indexmap = { version = "2.2.5", features = ["serde"]} tokio-native-tls = "^0" openssl = "=0.10" mail-parser = "^0" +matchit = "=0.7.3" datafusion = "39.0.0" object_store = { version = "0.10.0", features = ["aws", "azure"] } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c2d0f0dd53..531e0c3b9b 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -534f50aae2f89281304b3c3ba943d18afc07fad4 \ No newline at end of file +e093d51a219ce4ff0562a02db24ec402554a1f05 \ No newline at end of file diff --git a/backend/migrations/20240917172503_job_routing.down.sql b/backend/migrations/20240917172503_job_routing.down.sql new file mode 100644 index 0000000000..569385aa5a --- /dev/null +++ b/backend/migrations/20240917172503_job_routing.down.sql @@ -0,0 +1,7 @@ +-- Add down migration script here +DROP TABLE http_trigger; +DROP TYPE http_method; + +ALTER TABLE script DROP COLUMN has_preprocessor; + +DROP FUNCTION prevent_route_path_change(); \ No newline at end of file diff --git a/backend/migrations/20240917172503_job_routing.up.sql b/backend/migrations/20240917172503_job_routing.up.sql new file mode 100644 index 0000000000..42dd41e199 --- /dev/null +++ b/backend/migrations/20240917172503_job_routing.up.sql @@ -0,0 +1,61 @@ +-- Add up migration script here +CREATE TYPE http_method AS ENUM ('get', 'post', 'put', 'delete', 'patch'); + +CREATE TABLE http_trigger ( + path VARCHAR(255) NOT NULL, + route_path VARCHAR(255) NOT NULL, + route_path_key VARCHAR(255) NOT NULL, + script_path VARCHAR(255) NOT NULL, + is_flow BOOLEAN NOT NULL, + workspace_id VARCHAR(50) NOT NULL, + edited_by VARCHAR(50) NOT NULL, + email VARCHAR(255) NOT NULL, + edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + extra_perms JSONB NOT NULL DEFAULT '{}', + is_async BOOLEAN NOT NULL DEFAULT FALSE, + requires_auth BOOLEAN NOT NULL DEFAULT FALSE, + http_method http_method NOT NULL, + PRIMARY KEY (path, workspace_id) +); + + +GRANT SELECT, UPDATE ON http_trigger TO windmill_user; +GRANT ALL ON http_trigger TO windmill_admin; + +ALTER TABLE http_trigger ENABLE ROW LEVEL SECURITY; + +CREATE POLICY see_folder_extra_perms_user_select ON http_trigger FOR SELECT TO windmill_user +USING (SPLIT_PART(http_trigger.path, '/', 1) = 'f' AND SPLIT_PART(http_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[])); +CREATE POLICY see_folder_extra_perms_user_update ON http_trigger FOR UPDATE TO windmill_user +USING (SPLIT_PART(http_trigger.path, '/', 1) = 'f' AND SPLIT_PART(http_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[])); + + +CREATE POLICY see_extra_perms_user_select ON http_trigger FOR SELECT TO windmill_user +USING (extra_perms ? CONCAT('u/', current_setting('session.user'))); +CREATE POLICY see_extra_perms_user_update ON http_trigger FOR UPDATE TO windmill_user +USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean); + +CREATE POLICY see_extra_perms_groups_select ON http_trigger FOR SELECT TO windmill_user +USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]); +CREATE POLICY see_extra_perms_groups_update ON http_trigger FOR UPDATE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]) + AND value::boolean)); + +CREATE OR REPLACE FUNCTION prevent_route_path_change() +RETURNS TRIGGER AS $$ +BEGIN + IF CURRENT_USER <> 'windmill_admin' AND NEW.route_path <> OLD.route_path THEN + RAISE EXCEPTION 'Modification of route_path is only allowed by admins'; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER check_route_path_change +BEFORE UPDATE ON http_trigger +FOR EACH ROW +EXECUTE FUNCTION prevent_route_path_change(); + +ALTER TABLE script ADD COLUMN has_preprocessor BOOLEAN; diff --git a/backend/migrations/20240919200009_migrate_smtp_settings.down.sql b/backend/migrations/20240919200009_migrate_smtp_settings.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20240919200009_migrate_smtp_settings.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20240919200009_migrate_smtp_settings.up.sql b/backend/migrations/20240919200009_migrate_smtp_settings.up.sql new file mode 100644 index 0000000000..54ac58675e --- /dev/null +++ b/backend/migrations/20240919200009_migrate_smtp_settings.up.sql @@ -0,0 +1,8 @@ +-- Add up migration script here +INSERT INTO global_settings (name, value) +SELECT 'smtp_settings', config +FROM config +WHERE name = 'server' +ON CONFLICT (name) DO UPDATE SET value = excluded.value; + +DELETE FROM config WHERE name = 'server'; diff --git a/backend/parsers/windmill-parser-bash/src/lib.rs b/backend/parsers/windmill-parser-bash/src/lib.rs index 515ef3822a..9b62ad3b0c 100644 --- a/backend/parsers/windmill-parser-bash/src/lib.rs +++ b/backend/parsers/windmill-parser-bash/src/lib.rs @@ -16,7 +16,13 @@ pub fn parse_bash_sig(code: &str) -> anyhow::Result { let parsed = parse_bash_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing bash script".to_string())) } @@ -26,7 +32,13 @@ pub fn parse_powershell_sig(code: &str) -> anyhow::Result { let parsed = parse_powershell_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing powershell script".to_string())) } @@ -173,7 +185,8 @@ non_required="${5:-}" oidx: None } ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-go/src/lib.rs b/backend/parsers/windmill-parser-go/src/lib.rs index c2564dbbd2..218a6dc3ec 100644 --- a/backend/parsers/windmill-parser-go/src/lib.rs +++ b/backend/parsers/windmill-parser-go/src/lib.rs @@ -42,6 +42,7 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result { star_kwargs: false, args, no_main_func: Some(false), + has_preprocessor: None, }) } else { Ok(MainArgSignature { @@ -49,6 +50,7 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result { star_kwargs: false, args: vec![], no_main_func: Some(true), + has_preprocessor: None, }) } } @@ -241,7 +243,8 @@ func main(x int, y string, z bool, l []string, o struct { Name string `json:"nam oidx: None }, ], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-graphql/src/lib.rs b/backend/parsers/windmill-parser-graphql/src/lib.rs index 060ae294d1..a900c3244d 100644 --- a/backend/parsers/windmill-parser-graphql/src/lib.rs +++ b/backend/parsers/windmill-parser-graphql/src/lib.rs @@ -15,7 +15,13 @@ pub fn parse_graphql_sig(code: &str) -> anyhow::Result { let parsed = parse_graphql_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -119,7 +125,8 @@ query($i: Int, $arr: [String]!, $wahoo: String = "wahoo") { oidx: None } ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-php/src/lib.rs b/backend/parsers/windmill-parser-php/src/lib.rs index 5adf1dc0d7..4c239c21c9 100644 --- a/backend/parsers/windmill-parser-php/src/lib.rs +++ b/backend/parsers/windmill-parser-php/src/lib.rs @@ -82,6 +82,7 @@ pub fn parse_php_signature( star_kwargs: false, args, no_main_func: Some(false), + has_preprocessor: None, }) } else { Ok(MainArgSignature { @@ -89,6 +90,7 @@ pub fn parse_php_signature( star_kwargs: false, args: vec![], no_main_func: Some(true), + has_preprocessor: None, }) } } @@ -159,7 +161,8 @@ function main(string $input1 = \"hey\", bool $input2 = false, int $input3 = 3, f oidx: None } ], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 3adde32185..5b42171478 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -63,9 +63,17 @@ pub fn parse_python_signature( ) -> anyhow::Result { let main_name = override_main.unwrap_or("main".to_string()); + let has_preprocessor = !filter_non_main(code, "preprocessor").is_empty(); + let filtered_code = filter_non_main(code, &main_name); if filtered_code.is_empty() { - return Err(anyhow::anyhow!("No main function found".to_string(),)); + return Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![], + no_main_func: Some(true), + has_preprocessor: Some(has_preprocessor), + }); } let ast = Suite::parse(&filtered_code, "main.py") .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; @@ -119,6 +127,7 @@ pub fn parse_python_signature( }) .collect(), no_main_func: Some(false), + has_preprocessor: Some(has_preprocessor), }) } else { Ok(MainArgSignature { @@ -126,6 +135,7 @@ pub fn parse_python_signature( star_kwargs: false, args: vec![], no_main_func: Some(true), + has_preprocessor: Some(has_preprocessor), }) } } @@ -325,6 +335,7 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt }, ], no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -389,6 +400,7 @@ def main(test1: str, } ], no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -448,6 +460,7 @@ def main(test1: str, } ], no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -491,6 +504,7 @@ def main(test1: Literal["foo", "bar"], test2: List[Literal["foo", "bar"]]): retu } ], no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -521,6 +535,59 @@ def main(test1: DynSelect_foo): return oidx: None }], no_main_func: Some(false), + has_preprocessor: Some(false) + } + ); + + Ok(()) + } + + #[test] + fn test_parse_python_sig_6() -> anyhow::Result<()> { + let code = r#" + +import os + +def hello(): return + +"#; + //println!("{}", serde_json::to_string()?); + assert_eq!( + parse_python_signature(code, None)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![], + no_main_func: Some(true), + has_preprocessor: Some(false) + } + ); + + Ok(()) + } + + #[test] + fn test_parse_python_sig_7() -> anyhow::Result<()> { + let code = r#" + +import os + +def preprocessor(): return + +def main(): return + + + +"#; + //println!("{}", serde_json::to_string()?); + assert_eq!( + parse_python_signature(code, None)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![], + no_main_func: Some(false), + has_preprocessor: Some(true) } ); diff --git a/backend/parsers/windmill-parser-rust/src/lib.rs b/backend/parsers/windmill-parser-rust/src/lib.rs index f9a2a5b677..5d4d626f4c 100644 --- a/backend/parsers/windmill-parser-rust/src/lib.rs +++ b/backend/parsers/windmill-parser-rust/src/lib.rs @@ -29,6 +29,7 @@ pub fn parse_rust_signature(code: &str) -> anyhow::Result { star_kwargs: false, args, no_main_func: Some(false), + has_preprocessor: None, }) } else { Ok(MainArgSignature { @@ -36,6 +37,7 @@ pub fn parse_rust_signature(code: &str) -> anyhow::Result { star_kwargs: false, args: vec![], no_main_func: Some(true), + has_preprocessor: None, }) } } @@ -455,10 +457,7 @@ fn main( assert_eq!(ret.args[2].typ, Typ::Str(None)); assert_eq!(ret.args[3].name, "my_string_vec"); - assert_eq!( - ret.args[3].otyp, - Some("Vec < String >".to_string()) - ); + assert_eq!(ret.args[3].otyp, Some("Vec < String >".to_string())); assert_eq!(ret.args[3].typ, Typ::List(Box::new(Typ::Str(None)))); } diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index fbb4ac5d40..dd6bc2b955 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -20,7 +20,13 @@ pub fn parse_mysql_sig(code: &str) -> anyhow::Result { let parsed = parse_mysql_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -30,7 +36,13 @@ pub fn parse_pgsql_sig(code: &str) -> anyhow::Result { let parsed = parse_pg_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -40,7 +52,13 @@ pub fn parse_bigquery_sig(code: &str) -> anyhow::Result { let parsed = parse_bigquery_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -50,7 +68,13 @@ pub fn parse_snowflake_sig(code: &str) -> anyhow::Result { let parsed = parse_snowflake_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -60,7 +84,13 @@ pub fn parse_mssql_sig(code: &str) -> anyhow::Result { let parsed = parse_mssql_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -570,7 +600,8 @@ SELECT * FROM table WHERE token=$1::TEXT AND image=$2::BIGINT oidx: Some(2), }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -618,7 +649,8 @@ SELECT $2::TEXT; oidx: Some(3), }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -744,7 +776,8 @@ SELECT ?, ?; oidx: None, }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -791,7 +824,8 @@ SELECT :param2; oidx: None, }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -830,7 +864,8 @@ SELECT @token; oidx: None, }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -877,7 +912,8 @@ SELECT ?; oidx: None, } ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -924,7 +960,8 @@ SELECT @p2; oidx: None, }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 41ad759f0a..742a840802 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -159,6 +159,14 @@ pub fn parse_deno_signature( })? .body; + let has_preprocessor = ast.iter().any(|x| match x { + ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl { + decl: Decl::Fn(FnDecl { ident: Ident { sym, .. }, .. }), + .. + })) => &sym.to_string() == "preprocessor", + _ => false, + }); + let main_name = main_override.unwrap_or("main".to_string()); let params = ast.into_iter().find_map(|x| match x { ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl { @@ -178,6 +186,7 @@ pub fn parse_deno_signature( .map(|x| parse_param(x, &cm, skip_dflt, &mut c)) .collect::>>()?, no_main_func: Some(false), + has_preprocessor: Some(has_preprocessor), }; Ok(r) } else { @@ -186,6 +195,7 @@ pub fn parse_deno_signature( star_kwargs: false, args: vec![], no_main_func: Some(true), + has_preprocessor: Some(has_preprocessor), }) } } diff --git a/backend/parsers/windmill-parser-wasm/build-deno-pkgs.sh b/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh similarity index 64% rename from backend/parsers/windmill-parser-wasm/build-deno-pkgs.sh rename to backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh index 3ac0f9b424..7803fe22f0 100755 --- a/backend/parsers/windmill-parser-wasm/build-deno-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh @@ -10,35 +10,41 @@ wasm-pack build --release --target web --out-dir $OUT_DIR --all-features \ OUT_DIR="pkg-ts" wasm-pack build --release --target web --out-dir $OUT_DIR --features "ts-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-ts"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-ts"/' $OUT_DIR/package.json # sql languages, graphql and bash/powershell, since they all use regex OUT_DIR="pkg-regex" wasm-pack build --release --target web --out-dir $OUT_DIR \ --features "sql-parser,graphql-parser,bash-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-regex"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-regex"/' $OUT_DIR/package.json # python OUT_DIR="pkg-py" wasm-pack build --release --target web --out-dir $OUT_DIR --features "py-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-py"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-py"/' $OUT_DIR/package.json # go OUT_DIR="pkg-go" wasm-pack build --release --target web --out-dir $OUT_DIR --features "go-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-go"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-go"/' $OUT_DIR/package.json # php OUT_DIR="pkg-php" wasm-pack build --release --target web --out-dir $OUT_DIR --features "php-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json # rust OUT_DIR="pkg-rust" wasm-pack build --release --target web --out-dir $OUT_DIR --features "rust-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json + +# ansible +OUT_DIR="pkg-yaml" +wasm-pack build --release --target web --out-dir $OUT_DIR --features "ansible-parser" \ + -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-yaml"/' $OUT_DIR/package.json diff --git a/backend/parsers/windmill-parser-wasm/build.sh b/backend/parsers/windmill-parser-wasm/build.sh index 044f92d93b..b872e2c912 100755 --- a/backend/parsers/windmill-parser-wasm/build.sh +++ b/backend/parsers/windmill-parser-wasm/build.sh @@ -1,4 +1,4 @@ #!/bin/bash set -eou pipefail -deno task wasmbuild --out ../../../cli/wasm/ -p windmill-parser-wasm +deno task wasmbuild --out ../../../cli/wasm/ -p windmill-parser-wasm --all-features diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index c10f30c138..a5dd03ad91 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -17,8 +17,12 @@ fn wrap_sig(r: anyhow::Result) -> String { #[cfg(feature = "ts-parser")] #[wasm_bindgen] -pub fn parse_deno(code: &str) -> String { - wrap_sig(windmill_parser_ts::parse_deno_signature(code, false, None)) +pub fn parse_deno(code: &str, main_override: Option) -> String { + wrap_sig(windmill_parser_ts::parse_deno_signature( + code, + false, + main_override, + )) } #[cfg(feature = "ts-parser")] @@ -65,8 +69,11 @@ pub fn parse_go(code: &str) -> String { #[cfg(feature = "py-parser")] #[wasm_bindgen] -pub fn parse_python(code: &str) -> String { - wrap_sig(windmill_parser_py::parse_python_signature(code, None)) +pub fn parse_python(code: &str, main_override: Option) -> String { + wrap_sig(windmill_parser_py::parse_python_signature( + code, + main_override, + )) } #[cfg(feature = "sql-parser")] diff --git a/backend/parsers/windmill-parser-wasm/tests/wasm.rs b/backend/parsers/windmill-parser-wasm/tests/wasm.rs index 9f0feacbed..1c201ecd4e 100644 --- a/backend/parsers/windmill-parser-wasm/tests/wasm.rs +++ b/backend/parsers/windmill-parser-wasm/tests/wasm.rs @@ -139,7 +139,8 @@ export function main(test1?: string, test2: string = \"burkina\", oidx: None } ], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -215,7 +216,8 @@ export function main(test2 = \"burkina\", oidx: None } ], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -264,7 +266,8 @@ export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\", oidx: None } ], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -294,7 +297,8 @@ export function main(foo: (\"foo\" | \"bar\")[]) { has_default: false, oidx: None }], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -434,6 +438,7 @@ Write-Output 'Testing...' } ], no_main_func: None, + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index dd9e25e564..8c7068c54a 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -13,6 +13,7 @@ pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result anyhow::Result Typ { @@ -173,7 +180,12 @@ fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result anyhow::Result { if let Yaml::Hash(f) = yaml { - if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("resource".to_string())) - { + if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("resource".to_string())) { let target_path = f .get(&Yaml::String("target".to_string())) .and_then(|x| x.as_str()) @@ -262,7 +273,9 @@ fn parse_file_resource(yaml: &Yaml) -> anyhow::Result { ))?; return Ok(FileResource { resource_path: resource_path.clone(), target_path }); } - return Err(anyhow!("File resource should have a `resource` field, linking to a text file resource")); + return Err(anyhow!( + "File resource should have a `resource` field, linking to a text file resource" + )); } return Err(anyhow!("Invalid file resource: Should be a dictionary.")); } diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index c65b95d07c..6f793d0cc7 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -16,6 +16,7 @@ pub struct MainArgSignature { pub star_kwargs: bool, pub args: Vec, pub no_main_func: Option, + pub has_preprocessor: Option, } #[derive(Serialize, Clone, Debug, PartialEq)] @@ -153,7 +154,10 @@ mod test { #[test] fn test_mixed_delimiters() { - assert_eq!(to_snake_case("test-Case_with Spaces"), "test_case_with_spaces"); + assert_eq!( + to_snake_case("test-Case_with Spaces"), + "test_case_with_spaces" + ); } #[test] diff --git a/backend/src/main.rs b/backend/src/main.rs index 608e1870ec..20bc963bd5 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -8,7 +8,10 @@ use anyhow::Context; use git_version::git_version; -use monitor::{send_current_log_file_to_object_store, send_logs_to_object_store}; +use monitor::{ + reload_timeout_wait_result_setting, send_current_log_file_to_object_store, + send_logs_to_object_store, +}; use rand::Rng; use sqlx::{postgres::PgListener, Pool, Postgres}; use std::{ @@ -35,7 +38,7 @@ use windmill_common::{ JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, - SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, + SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, scripts::ScriptLang, stats_ee::schedule_stats, @@ -74,7 +77,7 @@ use crate::monitor::{ reload_critical_error_channels_setting, reload_extra_pip_index_url_setting, reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_jwt_secret_setting, reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting, - reload_retention_period_setting, reload_scim_token_setting, reload_server_config, + reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config, reload_worker_config, }; @@ -173,7 +176,7 @@ async fn cache_hub_scripts(file_path: Option) -> anyhow::Result<()> { None, &job_dir, "cache_init", - windmill_worker::get_common_bun_proc_envs("").await, + windmill_worker::get_common_bun_proc_envs(None).await, false, ) .await?; @@ -563,9 +566,7 @@ Windmill Community Edition {GIT_VERSION} "notify_config_change" => { match n.payload() { "server" if server_mode => { - tracing::info!("Server config change detected: {}", n.payload()); - - reload_server_config(&db).await; + tracing::error!("Server config change detected but server config is obsolete: {}", n.payload()); }, a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => { tracing::info!("Worker config change detected: {}", n.payload()); @@ -609,6 +610,12 @@ Windmill Community Edition {GIT_VERSION} tracing::error!("Error loading default tag per workspace workspaces: {e:#}"); } } + SMTP_SETTING => { + reload_smtp_config(&db).await; + }, + TIMEOUT_WAIT_RESULT_SETTING => { + reload_timeout_wait_result_setting(&db).await + }, RETENTION_PERIOD_SECS_SETTING => { reload_retention_period_setting(&db).await }, diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 39cfc1ce51..8397f134aa 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -22,6 +22,7 @@ use tokio::{ #[cfg(feature = "embedding")] use windmill_api::embeddings::update_embeddings_db; use windmill_api::{ + jobs::TIMEOUT_WAIT_RESULT, oauth2_ee::{build_oauth_clients, OAuthClient}, DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN, }; @@ -40,17 +41,17 @@ use windmill_common::{ KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, - SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, + SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, jobs::QueuedJob, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, - server::load_server_config, + server::load_smtp_config, tracing_init::JSON_FMT, users::truncate_token, utils::{now_from_db, rd_string, report_critical_error, Mode}, worker::{ load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting, - DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SERVER_CONFIG, WORKER_CONFIG, + DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG, WORKER_CONFIG, WORKER_GROUP, }, BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, @@ -172,8 +173,9 @@ pub async fn initial_load( reload_s3_cache_setting(&db).await; } + reload_smtp_config(&db).await; + if server_mode { - reload_server_config(&db).await; reload_retention_period_setting(&db).await; reload_request_size(&db).await; reload_saml_metadata_setting(&db).await; @@ -698,6 +700,15 @@ pub async fn reload_scim_token_setting(db: &DB) { .await; } +pub async fn reload_timeout_wait_result_setting(db: &DB) { + reload_option_setting_with_tracing( + db, + TIMEOUT_WAIT_RESULT_SETTING, + "TIMEOUT_WAIT_RESULT", + TIMEOUT_WAIT_RESULT.clone(), + ) + .await; +} pub async fn reload_saml_metadata_setting(db: &DB) { reload_option_setting_with_tracing( db, @@ -1129,14 +1140,14 @@ pub async fn expose_queue_metrics(db: &Pool) { } } -pub async fn reload_server_config(db: &Pool) { - let config = load_server_config(&db).await; - if let Err(e) = config { - tracing::error!("Error reloading server config: {:?}", e) +pub async fn reload_smtp_config(db: &Pool) { + let smtp_config = load_smtp_config(&db).await; + if let Err(e) = smtp_config { + tracing::error!("Error reloading smtp config: {:?}", e) } else { - let mut wc = SERVER_CONFIG.write().await; - tracing::info!("Reloading server config..."); - *wc = config.unwrap() + let mut wc = SMTP_CONFIG.write().await; + tracing::info!("Reloading smtp config..."); + *wc = smtp_config.unwrap() } } diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index ec4f4cd9f8..e92578b02f 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3294,6 +3294,7 @@ async fn run_deployed_relative_imports( visible_to_runner_only: None, no_main_func: None, codebase: None, + has_preprocessor: None, }, ) .await @@ -3324,6 +3325,7 @@ async fn run_deployed_relative_imports( dedicated_worker: None, language, priority: None, + apply_preprocessor: false, }) .push(&db2) .await; diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 8afb489378..fd8bedf0f9 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -94,6 +94,7 @@ object_store = { workspace = true, optional = true} openidconnect = { workspace = true, optional = true} url = { workspace = true, optional = true} jsonwebtoken = { workspace = true } +matchit.workspace = true pin-project.workspace = true crc.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 99e48ec00b..217fd46daf 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.397.1 + version: 1.398.1 title: Windmill API contact: @@ -742,6 +742,35 @@ paths: schema: type: string + /settings/test_critical_channels: + post: + summary: test critical channels + operationId: testCriticalChannels + tags: + - setting + requestBody: + description: test critical channel payload + required: true + content: + application/json: + schema: + type: array + items: + type: object + properties: + email: + type: string + slack_channel: + type: string + responses: + "200": + description: status + content: + text/plain: + schema: + type: string + + /settings/test_license_key: post: summary: test license key @@ -4182,6 +4211,11 @@ paths: in: query schema: type: integer + - name: skip_preprocessor + description: skip the preprocessor + in: query + schema: + type: boolean - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" @@ -5328,6 +5362,11 @@ paths: in: query schema: type: integer + - name: skip_preprocessor + description: skip the preprocessor + in: query + schema: + type: boolean - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/NewJobId" @@ -5436,6 +5475,11 @@ paths: in: query schema: type: integer + - name: skip_preprocessor + description: skip the preprocessor + in: query + schema: + type: boolean - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" @@ -6822,6 +6866,190 @@ paths: "201": description: default error handler set + + /w/{workspace}/http_triggers/create: + post: + summary: create http trigger + operationId: createHttpTrigger + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: new http trigger + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/NewHttpTrigger" + responses: + "201": + description: http trigger created + content: + text/plain: + schema: + type: string + + /w/{workspace}/http_triggers/update/{path}: + post: + summary: update http trigger + operationId: updateHttpTrigger + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/EditHttpTrigger" + responses: + "200": + description: http trigger updated + content: + text/plain: + schema: + type: string + + /w/{workspace}/http_triggers/delete/{path}: + delete: + summary: delete http trigger + operationId: deleteHttpTrigger + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: http trigger deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/http_triggers/get/{path}: + get: + summary: get http trigger + operationId: getHttpTrigger + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: http trigger deleted + content: + application/json: + schema: + $ref: "#/components/schemas/HttpTrigger" + + + /w/{workspace}/http_triggers/list: + get: + summary: list http triggers + operationId: listHttpTriggers + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + required: true + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + responses: + "200": + description: http trigger list + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/HttpTrigger" + + + /w/{workspace}/http_triggers/exists/{path}: + get: + summary: does http trigger exists + operationId: existsHttpTrigger + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: http trigger exists + content: + application/json: + schema: + type: boolean + + /w/{workspace}/http_triggers/route_exists: + post: + summary: does route exists + operationId: existsRoute + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: route exists request + required: true + content: + application/json: + schema: + type: object + properties: + route_path: + type: string + http_method: + type: string + enum: ["get", "post", "put", "delete", "patch"] + required: + - kind + - route_path + - http_method + responses: + "200": + description: route exists + content: + application/json: + schema: + type: boolean + + /w/{workspace}/http_triggers/used: + get: + summary: whether http triggers are used + operationId: used + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: whether http triggers are used + content: + application/json: + schema: + type: boolean + /groups/list: get: summary: list instance groups @@ -7648,6 +7876,7 @@ paths: folder, app, raw_app, + http_trigger, ] responses: "200": @@ -7684,6 +7913,7 @@ paths: folder, app, raw_app, + http_trigger, ] requestBody: description: acl to add @@ -7731,6 +7961,7 @@ paths: folder, app, raw_app, + http_trigger, ] requestBody: description: acl to add @@ -9421,6 +9652,8 @@ components: type: boolean codebase: type: string + has_preprocessor: + type: boolean required: - hash @@ -9438,6 +9671,7 @@ components: - kind - starred - no_main_func + - has_preprocessor NewScript: @@ -9519,6 +9753,8 @@ components: type: boolean codebase: type: string + has_preprocessor: + type: boolean required: - path - summary @@ -10319,6 +10555,9 @@ components: no_main_func: type: boolean nullable: true + has_preprocessor: + type: boolean + nullable: true required: - star_args - start_kwargs @@ -10326,6 +10565,7 @@ components: - type - error - no_main_func + - has_preprocessor Preview: type: object @@ -10715,6 +10955,122 @@ components: - is_flow - args + HttpTrigger: + type: object + properties: + path: + type: string + edited_by: + type: string + edited_at: + type: string + format: date-time + script_path: + type: string + route_path: + type: string + is_flow: + type: boolean + extra_perms: + type: object + additionalProperties: + type: boolean + email: + type: string + workspace_id: + type: string + http_method: + type: string + enum: + - get + - post + - put + - delete + - patch + is_async: + type: boolean + requires_auth: + type: boolean + + required: + - path + - edited_by + - edited_at + - script_path + - route_path + - extra_perms + - is_flow + - email + - workspace_id + - is_async + - requires_auth + - http_method + + NewHttpTrigger: + type: object + properties: + path: + type: string + script_path: + type: string + route_path: + type: string + is_flow: + type: boolean + http_method: + type: string + enum: + - get + - post + - put + - delete + - patch + is_async: + type: boolean + requires_auth: + type: boolean + + required: + - path + - script_path + - route_path + - is_flow + - is_async + - requires_auth + - http_method + + EditHttpTrigger: + type: object + properties: + path: + type: string + script_path: + type: string + route_path: + type: string + is_flow: + type: boolean + http_method: + type: string + enum: + - get + - post + - put + - delete + - patch + is_async: + type: boolean + requires_auth: + type: boolean + required: + - path + - script_path + - is_flow + - kind + - is_async + - requires_auth + - http_method + Group: type: object properties: diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 2fedf3709c..6e74866203 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -1258,6 +1258,7 @@ mod tests { delete_after_use: None, continue_on_error: None, })), + preprocessor_module: None, same_worker: false, concurrent_limit: None, concurrency_time_window_s: None, @@ -1330,7 +1331,8 @@ mod tests { "expr": "previous.isEmpty()", "skip_if_stopped": false } - } + }, + "preprocessor_module": Option::::None }); assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect)); } diff --git a/backend/windmill-api/src/granular_acls.rs b/backend/windmill-api/src/granular_acls.rs index fb6d2e26f5..ad5793816b 100644 --- a/backend/windmill-api/src/granular_acls.rs +++ b/backend/windmill-api/src/granular_acls.rs @@ -23,8 +23,17 @@ use windmill_common::{ utils::{not_found_if_none, StripPath}, }; -const KINDS: [&str; 9] = [ - "script", "group_", "resource", "schedule", "variable", "flow", "folder", "app", "raw_app", +const KINDS: [&str; 10] = [ + "script", + "group_", + "resource", + "schedule", + "variable", + "flow", + "folder", + "app", + "raw_app", + "http_trigger", ]; pub fn workspaced_service() -> Router { diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs new file mode 100644 index 0000000000..9e30743fbc --- /dev/null +++ b/backend/windmill-api/src/http_triggers.rs @@ -0,0 +1,651 @@ +use axum::{ + extract::{Path, Query}, + response::IntoResponse, + routing::{delete, get, post}, + Extension, Json, Router, +}; +use http::{HeaderMap, StatusCode}; +use serde::{Deserialize, Serialize}; +use sql_builder::{bind::Bind, SqlBuilder}; +use sqlx::prelude::FromRow; +use std::collections::HashMap; +use tower_http::cors::CorsLayer; +use windmill_audit::{audit_ee::audit_log, ActionKind}; +use windmill_common::{ + auth::fetch_authed_from_permissioned_as, + db::UserDB, + error::{self, JsonResult}, + users::username_to_permissioned_as, + utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, + worker::{to_raw_value, CLOUD_HOSTED}, +}; +use windmill_queue::PushArgsOwned; + +use crate::{ + db::{ApiAuthed, DB}, + jobs::{ + run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_flow_by_path_internal, + run_wait_result_script_by_path_internal, RunJobQuery, + }, + users::OptAuthed, +}; + +lazy_static::lazy_static! { + static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/:\w+").unwrap(); +} + +pub fn routes_global_service() -> Router { + let cors = CorsLayer::new() + .allow_methods([ + http::Method::GET, + http::Method::POST, + http::Method::DELETE, + http::Method::PUT, + http::Method::PATCH, + ]) + .allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION]) + .allow_origin(tower_http::cors::Any); + Router::new() + .route( + "/*path", + get(route_job) + .post(route_job) + .delete(route_job) + .put(route_job) + .patch(route_job) + .head(|| async { "" }), + ) + .layer(cors) +} + +pub fn workspaced_service() -> Router { + Router::new() + .route("/create", post(create_trigger)) + .route("/list", get(list_triggers)) + .route("/get/*path", get(get_trigger)) + .route("/update/*path", post(update_trigger)) + .route("/delete/*path", delete(delete_trigger)) + .route("/exists/*path", get(exists_trigger)) + .route("/used", get(used)) + .route("/route_exists", post(exists_route)) +} + +#[derive(Serialize, Deserialize, sqlx::Type)] +#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +enum HttpMethod { + Get, + Post, + Put, + Delete, + Patch, +} + +impl From for http::Method { + fn from(method: HttpMethod) -> Self { + match method { + HttpMethod::Get => http::Method::GET, + HttpMethod::Post => http::Method::POST, + HttpMethod::Put => http::Method::PUT, + HttpMethod::Delete => http::Method::DELETE, + HttpMethod::Patch => http::Method::PATCH, + } + } +} + +#[derive(Deserialize)] +struct NewTrigger { + path: String, + route_path: String, + script_path: String, + is_flow: bool, + is_async: bool, + requires_auth: bool, + http_method: HttpMethod, +} + +#[derive(FromRow, Serialize)] +struct Trigger { + workspace_id: String, + path: String, + route_path: String, + route_path_key: String, + script_path: String, + is_flow: bool, + edited_by: String, + email: String, + edited_at: chrono::DateTime, + extra_perms: serde_json::Value, + is_async: bool, + requires_auth: bool, + http_method: HttpMethod, +} + +#[derive(Deserialize)] +struct EditTrigger { + path: String, + route_path: Option, + script_path: String, + is_flow: bool, + is_async: bool, + requires_auth: bool, + http_method: HttpMethod, +} + +#[derive(Deserialize)] +pub struct ListTriggerQuery { + pub page: Option, + pub per_page: Option, + pub path: Option, + pub is_flow: Option, + pub path_start: Option, +} + +async fn list_triggers( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(lst): Query, +) -> error::JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let (per_page, offset) = paginate(Pagination { per_page: lst.per_page, page: lst.page }); + let mut sqlb = SqlBuilder::select_from("http_trigger") + .field("*") + .order_by("edited_at", true) + .and_where("workspace_id = ?".bind(&w_id)) + .offset(offset) + .limit(per_page) + .clone(); + if let Some(path) = lst.path { + sqlb.and_where_eq("script_path", "?".bind(&path)); + } + if let Some(is_flow) = lst.is_flow { + sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); + } + if let Some(path_start) = &lst.path_start { + sqlb.and_where_like_left("path", path_start); + } + let sql = sqlb + .sql() + .map_err(|e| error::Error::InternalErr(e.to_string()))?; + let rows = sqlx::query_as::<_, Trigger>(&sql) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(rows)) +} + +async fn get_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> error::JsonResult { + let mut tx = user_db.begin(&authed).await?; + let path = path.to_path(); + let trigger = sqlx::query_as!( + Trigger, + r#"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as "http_method: _", edited_by, email, edited_at, extra_perms, is_async, requires_auth + FROM http_trigger + WHERE workspace_id = $1 AND path = $2"#, + w_id, + path, + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + let trigger = not_found_if_none(trigger, "Trigger", path)?; + + Ok(Json(trigger)) +} + +async fn create_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(ct): Json, +) -> error::Result<(StatusCode, String)> { + require_admin(authed.is_admin, &authed.username)?; + + let route_path_key = ROUTE_PATH_KEY_RE.replace_all(ct.route_path.as_str(), ":key"); + + let mut tx = user_db.begin(&authed).await?; + sqlx::query!( + "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())", + w_id, + ct.path, + ct.route_path, + &route_path_key, + ct.script_path, + ct.is_flow, + ct.is_async, + ct.requires_auth, + ct.http_method as HttpMethod, + &authed.username, + &authed.email + ) + .execute(&mut *tx).await?; + + audit_log( + &mut *tx, + &authed, + "http_triggers.create", + ActionKind::Create, + &w_id, + Some(ct.path.as_str()), + None, + ) + .await?; + + tx.commit().await?; + + Ok((StatusCode::CREATED, format!("{}", ct.path))) +} + +async fn update_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(ct): Json, +) -> error::Result { + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + + if authed.is_admin { + if ct.route_path.is_none() { + return Err(error::Error::BadRequest( + "route_path is required".to_string(), + )); + } + + let route_path_key = + ROUTE_PATH_KEY_RE.replace_all(ct.route_path.as_ref().unwrap().as_str(), ":key"); + + sqlx::query!( + "UPDATE http_trigger + SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() + WHERE workspace_id = $11 AND path = $12", + ct.route_path, + &route_path_key, + ct.script_path, + ct.path, + ct.is_flow, + ct.http_method as HttpMethod, + &authed.username, + &authed.email, + ct.is_async, + ct.requires_auth, + w_id, + path, + ) + .execute(&mut *tx).await?; + } else { + sqlx::query!( + "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() + WHERE workspace_id = $9 AND path = $10", + ct.script_path, + ct.path, + ct.is_flow, + ct.http_method as HttpMethod, + &authed.username, + &authed.email, + ct.is_async, + ct.requires_auth, + w_id, + path, + ) + .execute(&mut *tx).await?; + } + + audit_log( + &mut *tx, + &authed, + "http_triggers.update", + ActionKind::Create, + &w_id, + Some(path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(path.to_string()) +} + +async fn delete_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + sqlx::query!( + "DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2", + w_id, + path, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "http_triggers.delete", + ActionKind::Delete, + &w_id, + Some(path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("HTTP trigger {path} deleted")) +} + +async fn used(Extension(db): Extension, Path(w_id): Path) -> JsonResult { + let used = sqlx::query_scalar!( + r#"SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1)"#, + w_id, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + Ok(Json(used)) +} + +async fn exists_trigger( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE path = $1 AND workspace_id = $2)", + path, + w_id, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + Ok(Json(exists)) +} + +#[derive(Deserialize)] +struct RouteExists { + route_path: String, + http_method: HttpMethod, +} + +async fn exists_route( + Extension(db): Extension, + Path(w_id): Path, + Json(RouteExists { route_path, http_method }): Json, +) -> JsonResult { + let route_path_key = ROUTE_PATH_KEY_RE.replace_all(route_path.as_str(), ":key"); + let exists = if *CLOUD_HOSTED { + sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3)", + &route_path_key, + w_id, + http_method as HttpMethod + ) + .fetch_one(&db) + .await? + .unwrap_or(false) + } else { + sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2)", + &route_path_key, + http_method as HttpMethod + ) + .fetch_one(&db) + .await? + .unwrap_or(false) + }; + Ok(Json(exists)) +} + +struct TriggerRoute { + path: String, + script_path: String, + is_flow: bool, + route_path: String, + workspace_id: String, + is_async: bool, + requires_auth: bool, + edited_by: String, + email: String, + http_method: HttpMethod, +} + +async fn fetch_api_authed( + username: String, + email: String, + w_id: &str, + db: &DB, + username_override: String, +) -> error::Result { + let permissioned_as = username_to_permissioned_as(username.as_str()); + let authed = + fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?; + Ok(ApiAuthed { + username: username, + email: email, + is_admin: authed.is_admin, + is_operator: authed.is_operator, + groups: authed.groups, + folders: authed.folders, + scopes: authed.scopes, + username_override: Some(username_override), + }) +} + +async fn get_http_route_trigger( + route_path: &str, + opt_authed: Option, + db: &DB, + user_db: UserDB, +) -> error::Result<(TriggerRoute, String, HashMap, ApiAuthed)> { + let (mut triggers, route_path) = if *CLOUD_HOSTED { + let mut splitted = route_path.split("/"); + let w_id = splitted.next().ok_or_else(|| { + error::Error::BadRequest("Missing workspace id in route path".to_string()) + })?; + let route_path = StripPath(splitted.collect::>().join("/")); + let triggers = sqlx::query_as!( + TriggerRoute, + r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger WHERE workspace_id = $1"#, + w_id + ) + .fetch_all(db) + .await?; + (triggers, route_path) + } else { + let triggers = sqlx::query_as!( + TriggerRoute, + r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger"#, + ) + .fetch_all(db) + .await?; + (triggers, StripPath(route_path.to_string())) + }; + + let mut router = matchit::Router::new(); + + for (idx, trigger) in triggers.iter().enumerate() { + let route_path = trigger.route_path.clone(); + router.insert(route_path.as_str(), idx).unwrap_or_else(|e| { + tracing::warn!( + "Failed to consider http trigger route {}: {:?}", + route_path, + e, + ); + }); + } + + let trigger_idx = router.at(route_path.0.as_str()).ok(); + + let matchit::Match { value: trigger_idx, params } = + not_found_if_none(trigger_idx, "Trigger", route_path.0.as_str())?; + + let trigger = triggers.remove(trigger_idx.to_owned()); + + let params: HashMap = params + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let username_override = if trigger.requires_auth { + if let Some(authed) = opt_authed { + // check that the user has access to the trigger + let mut tx = user_db.begin(&authed).await?; + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1 AND path = $2)", + trigger.workspace_id, + trigger.path + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + tx.commit().await?; + if exists { + Some(authed.display_username().to_owned()) + } else { + return Err(error::Error::NotAuthorized("Unauthorized".to_string())); + } + } else { + return Err(error::Error::NotAuthorized( + "Requires authentication".to_string(), + )); + } + } else { + None + }; + + let authed = fetch_api_authed( + trigger.edited_by.clone(), + trigger.email.clone(), + &trigger.workspace_id, + &db, + username_override.unwrap_or("anonymous".to_string()), + ) + .await?; + + Ok((trigger, route_path.0, params, authed)) +} + +async fn route_job( + Extension(db): Extension, + Extension(user_db): Extension, + Extension(rsmq): Extension>, + Path(route_path): Path, + OptAuthed(opt_authed): OptAuthed, + Query(query): Query>, + method: http::Method, + headers: HeaderMap, + mut args: PushArgsOwned, +) -> impl IntoResponse { + let route_path = route_path.to_path(); + let (trigger, called_path, params, authed) = + match get_http_route_trigger(route_path, opt_authed, &db, user_db.clone()).await { + Ok(trigger) => trigger, + Err(e) => return e.into_response(), + }; + let headers = headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) + .collect::>(); + let extra = args.extra.get_or_insert_with(HashMap::new); + extra.insert( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": "http", + "http": { + "route": trigger.route_path, + "path": called_path, + "method": method.to_string().to_lowercase(), + "params": params, + "query": query, + "headers": headers + }, + })), + ); + let http_method = http::Method::from(trigger.http_method); + + if http_method != method { + return error::Error::BadRequest("Invalid HTTP method".to_string()).into_response(); + } + + let label_prefix = Some(format!( + "http-{}-{}-", + http_method.as_str().to_lowercase(), + trigger.route_path + )); + + let run_query = RunJobQuery::default(); + + if trigger.is_flow { + if trigger.is_async { + run_flow_by_path_inner( + authed, + db, + user_db, + rsmq, + trigger.workspace_id.clone(), + StripPath(trigger.script_path.to_owned()), + run_query, + args, + label_prefix, + ) + .await + .into_response() + } else { + run_wait_result_flow_by_path_internal( + db, + run_query, + StripPath(trigger.script_path.to_owned()), + authed, + rsmq, + user_db, + args, + trigger.workspace_id.clone(), + label_prefix, + ) + .await + .into_response() + } + } else { + if trigger.is_async { + run_script_by_path_inner( + authed, + db, + user_db, + rsmq, + trigger.workspace_id.clone(), + StripPath(trigger.script_path.to_owned()), + run_query, + args, + label_prefix, + ) + .await + .into_response() + } else { + run_wait_result_script_by_path_internal( + db, + run_query, + StripPath(trigger.script_path.to_owned()), + authed, + rsmq, + user_db, + trigger.workspace_id.clone(), + args, + label_prefix, + ) + .await + .into_response() + } + } +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index cec301ad05..21b4ee16db 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -58,7 +58,7 @@ use tower_http::cors::{Any, CorsLayer}; use urlencoding::encode; use windmill_audit::audit_ee::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; -use windmill_common::worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, SERVER_CONFIG}; +use windmill_common::worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE}; use windmill_common::{ db::UserDB, error::{self, to_anyhow, Error}, @@ -556,9 +556,10 @@ pub async fn get_path_tag_limits_cache_for_hash<'c, R: rsmq_async::RsmqConnectio Option, Option, Option, + Option, )> { let script = sqlx::query!( - "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2", + "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where hash = $1 AND workspace_id = $2", hash, w_id ) @@ -583,6 +584,7 @@ pub async fn get_path_tag_limits_cache_for_hash<'c, R: rsmq_async::RsmqConnectio script.priority, script.delete_after_use, script.timeout, + script.has_preprocessor, )) } @@ -1058,6 +1060,7 @@ pub struct RunJobQuery { pub tag: Option, pub timeout: Option, pub cache_ttl: Option, + pub skip_preprocessor: Option, } impl RunJobQuery { @@ -2784,14 +2787,18 @@ pub async fn run_flow_by_path_inner( let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); - let (tag, dedicated_worker) = sqlx::query!( - "SELECT tag, dedicated_worker from flow WHERE path = $1 and workspace_id = $2", + let (tag, dedicated_worker, has_preprocessor) = sqlx::query!( + "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor + FROM flow + LEFT JOIN flow_version + ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + WHERE flow.path = $1 and flow.workspace_id = $2", flow_path, w_id ) .fetch_optional(&mut tx) .await? - .map(|x| (x.tag, x.dedicated_worker)) + .map(|x| (x.tag, x.dedicated_worker, x.has_preprocessor)) .ok_or_else(|| { Error::NotFound(format!( "flow not found at path {flow_path} in workspace {w_id}" @@ -2807,7 +2814,12 @@ pub async fn run_flow_by_path_inner( &db, tx, &w_id, - JobPayload::Flow { path: flow_path.to_string(), dedicated_worker }, + JobPayload::Flow { + path: flow_path.to_string(), + dedicated_worker, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), + }, PushArgs { args: &args.args, extra: args.extra }, &label_prefix .map(|x| x + authed.display_username()) @@ -2973,7 +2985,7 @@ pub async fn run_script_by_path_inner( let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); let (job_payload, tag, _delete_after_use, timeout) = - script_path_to_payload(script_path, &mut tx, &w_id).await?; + script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?; let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); @@ -3049,7 +3061,15 @@ pub async fn run_workflow_as_code( None, run_query.timeout, ), - JobKind::Script => script_path_to_payload(job.script_path(), &mut tx, &w_id).await?, + JobKind::Script => { + script_path_to_payload( + job.script_path(), + &mut tx, + &w_id, + run_query.skip_preprocessor, + ) + .await? + } _ => return Err(anyhow::anyhow!("Not supported").into()), }; @@ -3145,6 +3165,17 @@ impl Drop for Guard { } } +use std::sync::Arc; +use tokio::sync::RwLock; + +lazy_static::lazy_static! { + pub static ref TIMEOUT_WAIT_RESULT: Arc>> = Arc::new(RwLock::new( + std::env::var("TIMEOUT_WAIT_RESULT") + .ok() + .and_then(|x| x.parse::().ok()) + )); +} + #[derive(Deserialize)] pub struct WindmillCompositeResult { windmill_status_code: Option, @@ -3159,7 +3190,7 @@ async fn run_wait_result( username: &str, ) -> error::Result { let mut result = None; - let timeout = SERVER_CONFIG.read().await.timeout_wait_result.clone(); + let timeout = TIMEOUT_WAIT_RESULT.read().await.clone().unwrap_or(600); let timeout_ms = if timeout <= 0 { 2000 } else { @@ -3451,7 +3482,7 @@ pub async fn run_wait_result_job_by_path_get( let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); let (job_payload, tag, delete_after_use, timeout) = - script_path_to_payload(script_path, &mut tx, &w_id).await?; + script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag).await?; @@ -3527,7 +3558,7 @@ pub async fn run_wait_result_flow_by_path_get( let args = PushArgsOwned { extra: Some(payload_args), args: HashMap::new() }; run_wait_result_flow_by_path_internal( - db, run_query, flow_path, authed, rsmq, user_db, args, w_id, + db, run_query, flow_path, authed, rsmq, user_db, args, w_id, None, ) .await } @@ -3553,11 +3584,12 @@ pub async fn run_wait_result_script_by_path( user_db, w_id, args, + None, ) .await } -async fn run_wait_result_script_by_path_internal( +pub async fn run_wait_result_script_by_path_internal( db: sqlx::Pool, run_query: RunJobQuery, script_path: StripPath, @@ -3566,6 +3598,7 @@ async fn run_wait_result_script_by_path_internal( user_db: UserDB, w_id: String, args: PushArgsOwned, + label_prefix: Option, ) -> error::Result { check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; let script_path = script_path.to_path(); @@ -3574,7 +3607,7 @@ async fn run_wait_result_script_by_path_internal( let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); let (job_payload, tag, delete_after_use, timeout) = - script_path_to_payload(script_path, &mut tx, &w_id).await?; + script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag).await?; @@ -3587,7 +3620,9 @@ async fn run_wait_result_script_by_path_internal( &w_id, job_payload, PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), + &label_prefix + .map(|x| x + authed.display_username()) + .unwrap_or_else(|| authed.display_username().to_string()), &authed.email, username_to_permissioned_as(&authed.username), None, @@ -3644,6 +3679,7 @@ pub async fn run_wait_result_script_by_hash( priority, delete_after_use, timeout, + has_preprocessor, ) = get_path_tag_limits_cache_for_hash(&mut tx, &w_id, hash).await?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); @@ -3669,6 +3705,8 @@ pub async fn run_wait_result_script_by_hash( language, dedicated_worker, priority, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), }, PushArgs { args: &args.args, extra: args.extra }, authed.display_username(), @@ -3712,12 +3750,12 @@ pub async fn run_wait_result_flow_by_path( check_license_key_valid().await?; run_wait_result_flow_by_path_internal( - db, run_query, flow_path, authed, rsmq, user_db, args, w_id, + db, run_query, flow_path, authed, rsmq, user_db, args, w_id, None, ) .await } -async fn run_wait_result_flow_by_path_internal( +pub async fn run_wait_result_flow_by_path_internal( db: sqlx::Pool, run_query: RunJobQuery, flow_path: StripPath, @@ -3726,6 +3764,7 @@ async fn run_wait_result_flow_by_path_internal( user_db: UserDB, args: PushArgsOwned, w_id: String, + label_prefix: Option, ) -> error::Result { check_queue_too_long(&db, run_query.queue_limit).await?; @@ -3736,8 +3775,8 @@ async fn run_wait_result_flow_by_path_internal( let scheduled_for = run_query.get_scheduled_for(&db).await?; - let (tag, dedicated_worker, early_return) = sqlx::query!( - "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return + let (tag, dedicated_worker, early_return, has_preprocessor) = sqlx::query!( + "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] @@ -3747,7 +3786,7 @@ async fn run_wait_result_flow_by_path_internal( ) .fetch_optional(&mut tx) .await? - .map(|x| (x.tag, x.dedicated_worker, x.early_return)) + .map(|x| (x.tag, x.dedicated_worker, x.early_return, x.has_preprocessor)) .ok_or_else(|| { Error::NotFound(format!( "flow not found at path {flow_path} in workspace {w_id}" @@ -3763,9 +3802,16 @@ async fn run_wait_result_flow_by_path_internal( &db, tx, &w_id, - JobPayload::Flow { path: flow_path.to_string(), dedicated_worker }, + JobPayload::Flow { + path: flow_path.to_string(), + dedicated_worker, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), + }, PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), + &label_prefix + .map(|x| x + authed.display_username()) + .unwrap_or_else(|| authed.display_username().to_string()), &authed.email, username_to_permissioned_as(&authed.username), scheduled_for, @@ -4202,6 +4248,7 @@ async fn add_batch_jobs( _priority, _delete_after_use, timeout, + _, ) = get_latest_deployed_hash_for_path(&db, &w_id, &path).await?; ( Some(script_hash), @@ -4228,7 +4275,11 @@ async fn add_batch_jobs( JobPayload::RawFlow { value: fv.clone(), path: None, restarted_from: None } } else { if let Some(path) = batch_info.path.as_ref() { - JobPayload::Flow { path: path.to_string(), dedicated_worker: None } + JobPayload::Flow { + path: path.to_string(), + dedicated_worker: None, + apply_preprocessor: false, + } } else { Err(anyhow::anyhow!( "Path is required if no value is not provided" @@ -4450,6 +4501,7 @@ pub async fn run_job_by_hash_inner( priority, _delete_after_use, // not taken into account in async endpoints timeout, + has_preprocessor, ) = get_path_tag_limits_cache_for_hash(&mut tx, &w_id, hash).await?; check_scopes(&authed, || format!("run:script/{path}"))?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { @@ -4475,6 +4527,8 @@ pub async fn run_job_by_hash_inner( language, dedicated_worker, priority, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), }, PushArgs { args: &args.args, extra: args.extra }, &label_prefix @@ -4609,8 +4663,8 @@ async fn get_job_update( "progress_perc" ) - .fetch_one(&db) - .await? + .fetch_optional(&db) + .await?.and_then(|inner| inner) } else { None }; diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 4a37ad8f3e..b5bba9f645 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -60,6 +60,7 @@ mod flows; mod folders; mod granular_acls; mod groups; +mod http_triggers; mod indexer_ee; mod inputs; mod integration; @@ -283,7 +284,8 @@ pub async fn run_server( ) .nest("/variables", variables::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) - .nest("/oidc", oidc_ee::workspaced_service()), + .nest("/oidc", oidc_ee::workspaced_service()) + .nest("/http_triggers", http_triggers::workspaced_service()), ) .nest("/workspaces", workspaces::global_service()) .nest( @@ -336,7 +338,7 @@ pub async fn run_server( ) .nest( "/w/:workspace_id/capture_u", - capture::global_service().layer(cors), + capture::global_service().layer(cors.clone()), ) .nest( "/auth", @@ -346,6 +348,10 @@ pub async fn run_server( "/oauth", oauth2_ee::global_service().layer(Extension(Arc::clone(&sp_extension))), ) + .nest( + "/r", + http_triggers::routes_global_service().layer(from_extractor::()), + ) .route("/version", get(git_v)) .route("/uptodate", get(is_up_to_date)) .route("/ee_license", get(ee_license)) diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 61e8c553ee..aedb16bb15 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -103,6 +103,8 @@ pub struct ScriptWDraft { pub visible_to_runner_only: Option, #[serde(skip_serializing_if = "Option::is_none")] pub no_main_func: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_preprocessor: Option, } pub fn global_service() -> Router { @@ -581,6 +583,7 @@ async fn create_script_internal<'c>( || ns.language == ScriptLang::Bunnative || ns.language == ScriptLang::Deno || ns.language == ScriptLang::Rust + || ns.language == ScriptLang::Ansible || ns.language == ScriptLang::Php) { Some(String::new()) @@ -612,8 +615,8 @@ async fn create_script_internal<'c>( content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)", + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31)", &w_id, &hash.0, ns.path, @@ -643,7 +646,8 @@ async fn create_script_internal<'c>( ns.concurrency_key, ns.visible_to_runner_only, ns.no_main_func, - codebase + codebase, + ns.has_preprocessor, ) .execute(&mut tx) .await?; @@ -738,6 +742,7 @@ async fn create_script_internal<'c>( }, ); } + let permissioned_as = username_to_permissioned_as(&authed.username); if needs_lock_gen { let tag = if ns.dedicated_worker.is_some_and(|x| x) { @@ -878,7 +883,7 @@ async fn get_script_by_path_w_draft( let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 \ AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \ diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 3b58118d45..395387ebd6 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -24,6 +24,8 @@ use axum::{ #[cfg(feature = "enterprise")] use axum::extract::Query; +#[cfg(feature = "enterprise")] +use windmill_common::ee::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel}; use serde::Deserialize; use windmill_common::{ error::{self, JsonResult, Result}, @@ -55,7 +57,8 @@ pub fn global_service() -> Router { get(get_latest_key_renewal_attempt), ) .route("/renew_license_key", post(renew_license_key)) - .route("/customer_portal", post(create_customer_portal_session)); + .route("/customer_portal", post(create_customer_portal_session)) + .route("/test_critical_channels", post(test_critical_channels)); #[cfg(feature = "parquet")] { @@ -387,3 +390,27 @@ pub async fn create_customer_portal_session( return Ok(url); } + +#[cfg(feature = "enterprise")] +pub async fn test_critical_channels( + Extension(db): Extension, + authed: ApiAuthed, + Json(test_critical_channels): Json>, +) -> Result { + require_super_admin(&db, &authed.email).await?; + + #[cfg(feature = "enterprise")] + send_critical_alert( + "Test critical error".to_string(), + &db, + CriticalAlertKind::CriticalError, + Some(test_critical_channels), + ) + .await; + Ok("Sent test critical error".to_string()) +} + +#[cfg(not(feature = "enterprise"))] +pub async fn test_critical_channels() -> Result { + Ok("Critical channels require EE".to_string()) +} diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index b48e1391e4..7be613a93e 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -51,7 +51,7 @@ use windmill_audit::ActionKind; use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING; use windmill_common::users::truncate_token; use windmill_common::utils::{paginate, send_email}; -use windmill_common::worker::{CLOUD_HOSTED, SERVER_CONFIG}; +use windmill_common::worker::{CLOUD_HOSTED, SMTP_CONFIG}; use windmill_common::{ auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, JWT_SECRET}, db::UserDB, @@ -1966,7 +1966,7 @@ pub fn send_email_if_possible(subject: &str, content: &str, to: &str) { } pub async fn send_email_if_possible_intern(subject: &str, content: &str, to: String) -> Result<()> { - if let Some(smtp) = SERVER_CONFIG.read().await.smtp.clone() { + if let Some(smtp) = SMTP_CONFIG.read().await.clone() { send_email(subject, content, vec![to], smtp, None).await?; } return Ok(()); diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index e32953722d..7306063856 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -2317,6 +2317,8 @@ struct ScriptMetadata { pub codebase: Option, #[serde(skip_serializing_if = "Option::is_none")] pub concurrency_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_preprocessor: Option, } pub fn is_none_or_false(val: &Option) -> bool { @@ -2602,6 +2604,7 @@ async fn tarball_workspace( no_main_func: script.no_main_func, codebase: script.codebase, concurrency_key: script.concurrency_key, + has_preprocessor: script.has_preprocessor, }; let metadata_str = serde_json::to_string_pretty(&metadata).unwrap(); archive diff --git a/backend/windmill-common/src/ee.rs b/backend/windmill-common/src/ee.rs index 9b44ddad26..482b61a0fe 100644 --- a/backend/windmill-common/src/ee.rs +++ b/backend/windmill-common/src/ee.rs @@ -36,7 +36,13 @@ pub enum CriticalAlertKind { } #[cfg(feature = "enterprise")] -pub async fn send_critical_alert(_error_message: String, _db: &DB, _kind: CriticalAlertKind) {} +pub async fn send_critical_alert( + _error_message: String, + _db: &DB, + _kind: CriticalAlertKind, + _channels: Option>, +) { +} #[cfg(feature = "enterprise")] pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () { diff --git a/backend/windmill-common/src/flow_status.rs b/backend/windmill-common/src/flow_status.rs index 946cd45bff..035cadecd8 100644 --- a/backend/windmill-common/src/flow_status.rs +++ b/backend/windmill-common/src/flow_status.rs @@ -29,6 +29,7 @@ pub struct FlowStatus { pub step: i32, pub modules: Vec, pub failure_module: Box, + pub preprocessor_module: Option, #[serde(skip_serializing_if = "HashMap::is_empty")] #[serde(default)] @@ -355,7 +356,11 @@ impl FlowStatusModule { impl FlowStatus { pub fn new(f: &FlowValue) -> Self { Self { - step: 0, + step: if f.preprocessor_module.is_some() { + -1 + } else { + 0 + }, approval_conditions: None, modules: f .modules @@ -372,6 +377,13 @@ impl FlowStatus { .unwrap_or_else(|| "failure".to_string()), }, }), + preprocessor_module: if f.preprocessor_module.is_some() { + Some(FlowStatusModule::WaitingForPriorSteps { + id: f.preprocessor_module.as_ref().unwrap().id.clone(), + }) + } else { + None + }, cleanup_module: FlowCleanupModule { flow_jobs_to_clean: vec![] }, retry: RetryStatus { fail_count: 0, failed_jobs: vec![] }, restarted_from: None, diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 3d64dc09da..b1ad85da99 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -102,6 +102,8 @@ pub struct FlowValue { #[serde(default)] pub failure_module: Option>, #[serde(default)] + pub preprocessor_module: Option>, + #[serde(default)] #[serde(skip_serializing_if = "is_default")] pub same_worker: bool, diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index b5d6e6da4d..36b80c9fd7 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -14,6 +14,8 @@ pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url"; pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url"; pub const SCIM_TOKEN_SETTING: &str = "scim_token"; pub const SAML_METADATA_SETTING: &str = "saml_metadata"; +pub const SMTP_SETTING: &str = "smtp_settings"; +pub const TIMEOUT_WAIT_RESULT_SETTING: &str = "timeout_wait_result"; pub const UNIQUE_ID_SETTING: &str = "uid"; pub const DISABLE_STATS_SETTING: &str = "disable_stats"; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index e9c099a73c..f7cfe1c9ee 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -11,6 +11,8 @@ use uuid::Uuid; pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; +pub const PREPROCESSOR_FAKE_ENTRYPOINT: &str = "__WM_PREPROCESSOR"; + use crate::{ error::{self, to_anyhow, Error}, flow_status::{FlowStatus, RestartedFrom}, @@ -302,6 +304,7 @@ pub enum JobPayload { dedicated_worker: Option, language: ScriptLang, priority: Option, + apply_preprocessor: bool, }, Code(RawCode), Dependencies { @@ -331,6 +334,7 @@ pub enum JobPayload { Flow { path: String, dedicated_worker: Option, + apply_preprocessor: bool, }, RestartedFlow { completed_job_id: Uuid, @@ -383,6 +387,7 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre script_path: &str, db: E, w_id: &str, + skip_preprocessor: Option, ) -> error::Result<(JobPayload, Option, Option, Option)> { let (job_payload, tag, delete_after_use, script_timeout) = if script_path.starts_with("hub/") { ( @@ -404,6 +409,7 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre priority, delete_after_use, script_timeout, + has_preprocessor, ) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?; ( JobPayload::ScriptHash { @@ -416,6 +422,8 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre language, dedicated_worker, priority, + apply_preprocessor: !skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), }, tag, delete_after_use, @@ -473,7 +481,7 @@ pub async fn get_payload_tag_from_prefixed_path<'e, E: sqlx::Executor<'e, Databa w_id: &str, ) -> Result<(JobPayload, Option), Error> { let (payload, tag, _, _) = if path.starts_with("script/") { - script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id).await? + script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id, Some(true)).await? } else if path.starts_with("flow/") { let path = path.strip_prefix("flow/").unwrap().to_string(); let r = sqlx::query!( @@ -486,7 +494,12 @@ pub async fn get_payload_tag_from_prefixed_path<'e, E: sqlx::Executor<'e, Databa let (tag, dedicated_worker) = r .map(|x| (x.tag, x.dedicated_worker)) .unwrap_or_else(|| (None, None)); - (JobPayload::Flow { path, dedicated_worker }, tag, None, None) + ( + JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false }, + tag, + None, + None, + ) } else { return Err(Error::BadRequest(format!( "path must start with script/ or flow/ (got {})", diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index ecdc4cd218..48c31fbebb 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -271,9 +271,10 @@ pub async fn get_latest_deployed_hash_for_path<'e, E: sqlx::Executor<'e, Databas Option, Option, Option, + Option, )> { let r_o = sqlx::query!( - "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where path = $1 AND workspace_id = $2 AND + "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where path = $1 AND workspace_id = $2 AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", script_path, @@ -296,6 +297,7 @@ pub async fn get_latest_deployed_hash_for_path<'e, E: sqlx::Executor<'e, Databas script.priority, script.delete_after_use, script.timeout, + script.has_preprocessor, )) } diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 5899b35575..8732e6d221 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -193,6 +193,8 @@ pub struct Script { pub no_main_func: Option, #[serde(skip_serializing_if = "Option::is_none")] pub codebase: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_preprocessor: Option, } #[derive(Serialize, sqlx::FromRow)] @@ -289,6 +291,7 @@ pub struct NewScript { pub visible_to_runner_only: Option, pub no_main_func: Option, pub codebase: Option, + pub has_preprocessor: Option, } fn lock_deserialize<'de, D>(deserializer: D) -> Result, D::Error> diff --git a/backend/windmill-common/src/server.rs b/backend/windmill-common/src/server.rs index f18d3f873c..9fb39ad18b 100644 --- a/backend/windmill-common/src/server.rs +++ b/backend/windmill-common/src/server.rs @@ -13,22 +13,20 @@ pub struct Smtp { } #[derive(Serialize, Deserialize, PartialEq)] -pub struct ServerConfigOpt { +pub struct SmtpConfigOpt { pub smtp_host: Option, pub smtp_username: Option, pub smtp_password: Option, pub smtp_port: Option, pub smtp_from: Option, pub smtp_tls_implicit: Option, - pub timeout_wait_result: Option, } -pub async fn load_server_config(db: &DB) -> error::Result { - let config: ServerConfigOpt = - sqlx::query_scalar!("SELECT config FROM config WHERE name = 'server'",) +pub async fn load_smtp_config(db: &DB) -> error::Result> { + let config: SmtpConfigOpt = + sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'smtp_settings'",) .fetch_optional(db) .await? - .flatten() .map(|x| serde_json::from_value(x).ok()) .flatten() .unwrap_or_default(); @@ -77,20 +75,10 @@ pub async fn load_server_config(db: &DB) -> error::Result { tracing::warn!("SMTP not configured"); } - Ok(ServerConfig { - smtp, - timeout_wait_result: config - .timeout_wait_result - .ok_or( - std::env::var("TIMEOUT_WAIT_RESULT") - .ok() - .and_then(|x| x.parse::().ok()), - ) - .unwrap_or(600), - }) + Ok(smtp) } -impl Default for ServerConfigOpt { +impl Default for SmtpConfigOpt { fn default() -> Self { Self { smtp_from: None, @@ -99,13 +87,6 @@ impl Default for ServerConfigOpt { smtp_port: None, smtp_tls_implicit: None, smtp_username: None, - timeout_wait_result: Default::default(), } } } - -#[derive(PartialEq, Clone, Debug)] -pub struct ServerConfig { - pub smtp: Option, - pub timeout_wait_result: u64, -} diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 6703078f20..8bbe541f07 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -288,7 +288,7 @@ pub async fn report_critical_error(error_message: String, _db: DB) -> () { } #[cfg(feature = "enterprise")] - send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError).await; + send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError, None).await; } pub async fn report_recovered_critical_error(message: String, _db: DB) -> () { @@ -304,5 +304,11 @@ pub async fn report_recovered_critical_error(message: String, _db: DB) -> () { tracing::error!("Failed to save critical error to database: {}", err); } #[cfg(feature = "enterprise")] - send_critical_alert(message, &_db, CriticalAlertKind::RecoveredCriticalError).await; + send_critical_alert( + message, + &_db, + CriticalAlertKind::RecoveredCriticalError, + None, + ) + .await; } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index ce07438dd5..bf355f92e1 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -14,7 +14,7 @@ use std::{ }; use tokio::sync::RwLock; -use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::ServerConfig, DB}; +use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::Smtp, DB}; lazy_static::lazy_static! { pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string()); @@ -66,8 +66,7 @@ lazy_static::lazy_static! { pub static ref WORKER_SUSPENDED_PULL_QUERY: Arc> = Arc::new(RwLock::new("".to_string())); - pub static ref SERVER_CONFIG: Arc> = Arc::new(RwLock::new(ServerConfig { smtp: Default::default(), timeout_wait_result: 20 })); - + pub static ref SMTP_CONFIG: Arc>> = Arc::new(RwLock::new(None)); pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok(); @@ -195,7 +194,11 @@ fn normalize_path(path: &Path) -> PathBuf { } ret } -pub fn write_file_at_user_defined_location(job_dir: &str, user_defined_path: &str, content: &str) -> error::Result { +pub fn write_file_at_user_defined_location( + job_dir: &str, + user_defined_path: &str, + content: &str, +) -> error::Result { let job_dir = Path::new(job_dir); let user_path = PathBuf::from(user_defined_path); @@ -210,9 +213,11 @@ pub fn write_file_at_user_defined_location(job_dir: &str, user_defined_path: &st return Err(std::io::Error::new( std::io::ErrorKind::PermissionDenied, "Path is outside the allowed job directory.", - ).into()); + ) + .into()); } + let full_path = normalized_full_path.as_path(); if let Some(parent_dir) = full_path.parent() { std::fs::create_dir_all(parent_dir)?; } @@ -220,7 +225,7 @@ pub fn write_file_at_user_defined_location(job_dir: &str, user_defined_path: &st let mut file = File::create(full_path)?; file.write_all(content.as_bytes())?; file.flush()?; - Ok(file) + Ok(normalized_full_path) } pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> { diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 8a6fdcbab7..68f415e7cf 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -52,6 +52,7 @@ use windmill_common::{ }, jobs::{ get_payload_tag_from_prefixed_path, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode, + ENTRYPOINT_OVERRIDE, PREPROCESSOR_FAKE_ENTRYPOINT, }, schedule::Schedule, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, @@ -1021,6 +1022,7 @@ pub async fn add_completed_job< .clone() .unwrap_or_else(|| ScriptLang::Deno), priority: queued_job.priority, + apply_preprocessor: false, }, queued_job .args @@ -2864,10 +2866,21 @@ where (content_type, extra, raw, wrap_body) }; - if content_type.is_none() || content_type.unwrap().starts_with("application/json") { + let no_content_type = content_type.is_none(); + if no_content_type || content_type.unwrap().starts_with("application/json") { let bytes = Bytes::from_request(req, _state) .await .map_err(IntoResponse::into_response)?; + if no_content_type && bytes.is_empty() { + if use_raw { + extra.insert("raw_string".to_string(), to_raw_value(&"".to_string())); + } + let mut args = HashMap::new(); + if wrap_body { + args.insert("body".to_string(), to_raw_value(&serde_json::json!({}))); + } + return Ok(PushArgsOwned { extra: Some(extra), args: args }); + } let str = String::from_utf8(bytes.to_vec()) .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; @@ -2992,7 +3005,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( mut tx: PushIsolationLevel<'c, R>, workspace_id: &str, job_payload: JobPayload, - args: PushArgs<'d>, + mut args: PushArgs<'d>, user: &str, mut email: &str, mut permissioned_as: String, @@ -3211,21 +3224,38 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( language, dedicated_worker, priority, - } => ( - Some(hash.0), - Some(path), - None, - JobKind::Script, - None, - None, - Some(language), - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - cache_ttl, - dedicated_worker, - priority, - ), + apply_preprocessor, + } => { + let extra = args.extra.get_or_insert_with(HashMap::new); + if apply_preprocessor { + extra.insert( + ENTRYPOINT_OVERRIDE.to_string(), + to_raw_value(&PREPROCESSOR_FAKE_ENTRYPOINT), + ); + extra.entry("wm_trigger".to_string()).or_insert_with(|| { + to_raw_value(&serde_json::json!({ + "kind": "webhook", + })) + }); + } else { + extra.remove("wm_trigger"); + } + ( + Some(hash.0), + Some(path), + None, + JobKind::Script, + None, + None, + Some(language), + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + cache_ttl, + dedicated_worker, + priority, + ) + } JobPayload::ScriptHub { path } => { if path == "hub/7771/slack" || path == "hub/7836/slack" { permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string(); @@ -3404,9 +3434,13 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( branch_or_iteration_n: restarted_from_val.branch_or_iteration_n, }), user_states, + preprocessor_module: None, } } - _ => FlowStatus::new(&value), // this is a new flow being pushed, flow_status is set to flow_value + _ => { + value.preprocessor_module = None; + FlowStatus::new(&value) + } // this is a new flow being pushed, flow_status is set to flow_value }; ( None, @@ -3473,6 +3507,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( early_return: None, concurrency_key: custom_concurrency_key.clone(), priority: priority, + preprocessor_module: None, }; ( None, @@ -3490,7 +3525,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( priority, ) } - JobPayload::Flow { path, dedicated_worker } => { + JobPayload::Flow { path, dedicated_worker, apply_preprocessor } => { let value_json = fetch_scalar_isolated!( sqlx::query_as::<_, FlowRawValue>( "SELECT flow_version.value FROM flow @@ -3518,6 +3553,18 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( let custom_concurrency_key = value.concurrency_key.clone(); let concurrency_time_window_s = value.concurrency_time_window_s.clone(); let concurrent_limit = value.concurrent_limit.clone(); + + let extra = args.extra.get_or_insert_with(HashMap::new); + if !apply_preprocessor { + value.preprocessor_module = None; + extra.remove("wm_trigger"); + } else { + extra.entry("wm_trigger".to_string()).or_insert_with(|| { + to_raw_value(&serde_json::json!({ + "kind": "webhook", + })) + }); + } let status = Some(FlowStatus::new(&value)); ( None, @@ -3574,6 +3621,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( branch_or_iteration_n, }), user_states, + preprocessor_module: None, }; ( None, diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index aacfa87c10..d39053c3c7 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -120,7 +120,11 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>( .map(|x| (x.tag, x.dedicated_worker)) .unwrap_or_else(|| (None, None)); ( - JobPayload::Flow { path: schedule.script_path.clone(), dedicated_worker }, + JobPayload::Flow { + path: schedule.script_path.clone(), + dedicated_worker, + apply_preprocessor: false, + }, tag, None, ) @@ -188,6 +192,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>( dedicated_worker, language, priority, + apply_preprocessor: false, }, if schedule.tag.as_ref().is_some_and(|x| x != "") { schedule.tag.clone() diff --git a/backend/windmill-worker/nsjail/run.ansible.config.proto b/backend/windmill-worker/nsjail/run.ansible.config.proto new file mode 100644 index 0000000000..463de39770 --- /dev/null +++ b/backend/windmill-worker/nsjail/run.ansible.config.proto @@ -0,0 +1,140 @@ +name: "ansible run script" + +mode: ONCE +hostname: "ansible" +log_level: ERROR + +rlimit_as: 4096 +rlimit_cpu: 1000 +rlimit_fsize: 1000 +rlimit_nofile: 10000 + +cwd: "/tmp" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +keep_caps: false +keep_env: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + dst: "/dev/shm" + fstype: "tmpfs" + rw: true + is_bind: false +} + +mount { + dst: "/tmp" + fstype: "tmpfs" + rw: true + options: "size=800000000" +} + +mount { + src: "{JOB_DIR}/main.yml" + dst: "/tmp/main.yml" + is_bind: true +} + +mount { + src: "{JOB_DIR}/wrapper.sh" + dst: "/tmp/wrapper.sh" + is_bind: true +} + +mount { + src: "{JOB_DIR}/requirements.yml" + dst: "/tmp/requirements.yml" + is_bind: true + mandatory: false +} + +mount { + src: "{JOB_DIR}/ansible.cfg" + dst: "/tmp/ansible.cfg" + is_bind: true +} + +mount { + src: "{JOB_DIR}/ansible_collections/" + dst: "/tmp/ansible_collections/" + is_bind: true +} + +mount { + src: "{JOB_DIR}/args.json" + dst: "/tmp/args.json" + is_bind: true +} + +mount { + src: "{JOB_DIR}/result.json" + dst: "/tmp/result_nsjail_mount.json" + rw: true + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +{SHARED_MOUNT} + +{SHARED_DEPENDENCIES} + +{FILE_RESOURCES} + +iface_no_lo: true + +envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" +envar: "PYTHONPATH={ADDITIONAL_PYTHON_PATHS}" +envar: "HOME=/tmp" +envar: "ANSIBLE_CONFIG=/tmp/ansible.cfg" diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 785edfa85b..5130a41614 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -1,6 +1,9 @@ -use std::{collections::HashMap, process::Stdio}; +use std::{ + collections::HashMap, os::unix::fs::PermissionsExt, path::{Path, PathBuf}, process::Stdio +}; use anyhow::anyhow; +use itertools::Itertools; use serde_json::value::RawValue; use tokio::process::Command; use uuid::Uuid; @@ -13,12 +16,10 @@ use windmill_parser_yaml::AnsibleRequirements; use windmill_queue::{append_logs, CanceledBy}; use crate::{ - common::{ - get_reserved_variables, handle_child, read_and_check_result, - start_child_process, transform_json, - }, - python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}, - AuthedClientBackgroundTask, DISABLE_NSJAIL, HOME_ENV, PATH_ENV, TZ_ENV, + bash_executor::BIN_BASH, common::{ + get_reserved_variables, handle_child, read_and_check_result, start_child_process, + transform_json, + }, python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV }; lazy_static::lazy_static! { @@ -29,6 +30,8 @@ lazy_static::lazy_static! { std::env::var("ANSIBLE_GALAXY_PATH").unwrap_or("/usr/local/bin/ansible-galaxy".to_string()); } +const NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT: &str = include_str!("../nsjail/run.ansible.config.proto"); + async fn handle_ansible_python_deps( job_dir: &str, requirements_o: Option, @@ -98,6 +101,7 @@ async fn handle_ansible_python_deps( } Ok(additional_python_paths) } + async fn install_galaxy_collections( collections_yml: &str, job_dir: &str, @@ -155,6 +159,24 @@ async fn install_galaxy_collections( Ok(()) } +#[cfg(not(feature = "enterprise"))] +fn check_ansible_exists() -> Result<(), error::Error> { + if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() { + let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); + return Err(error::Error::NotFound(msg)); + } + Ok(()) +} + +#[cfg(feature = "enterprise")] +fn check_ansible_exists() -> Result<(), error::Error> { + if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() { + let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str()); + return Err(error::Error::NotFound(msg)); + } + Ok(()) +} + pub async fn handle_ansible_job( requirements_o: Option, job_dir: &str, @@ -166,19 +188,16 @@ pub async fn handle_ansible_job( db: &sqlx::Pool, client: &AuthedClientBackgroundTask, inner_content: &String, + shared_mount: &str, base_internal_url: &str, envs: HashMap, ) -> windmill_common::error::Result> { + check_ansible_exists()?; + let (logs, reqs, playbook) = windmill_parser_yaml::parse_ansible_reqs(inner_content)?; append_logs(&job.id, &job.workspace_id, logs, db).await; write_file(job_dir, "main.yml", &playbook)?; - let ansible_cfg_content = r#" -[defaults] -collections_path = ./ -roles_path = ./roles -"#; - write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?; let additional_python_paths = handle_ansible_python_deps( job_dir, @@ -215,6 +234,7 @@ roles_path = ./roles interpolated_args = None; write_file(job_dir, "args.json", "{}")?; }; + write_file(job_dir, "result.json", "")?; let inventories: Vec = reqs .as_ref() @@ -228,8 +248,9 @@ roles_path = ./roles .unwrap_or_else(|| vec![]); let authed_client = client.get_authed().await; + let mut nsjail_extra_mounts = vec![]; if let Some(r) = reqs { - create_file_resources( + nsjail_extra_mounts = create_file_resources( &job.id, &job.workspace_id, job_dir, @@ -261,18 +282,97 @@ roles_path = ./roles db, ) .await; - - if !*DISABLE_NSJAIL { - return Err(anyhow!("Ansible is not supported with nsjail, disable nsjail on your worker to run ansible playbooks").into()); - } + let ansible_cfg_content = format!(r#" +[defaults] +collections_path = ./ +roles_path = ./roles +home={job_dir}/.ansible +local_tmp={job_dir}/.ansible/tmp +remote_tmp={job_dir}/.ansible/tmp +"#); + write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?; let mut reserved_variables = get_reserved_variables(job, &authed_client.token, db).await?; let additional_python_paths_folders = additional_python_paths.join(":"); - reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders); + + if !*DISABLE_NSJAIL { + let shared_deps = additional_python_paths + .into_iter() + .map(|pp| { + format!( + r#" +mount {{ + src: "{pp}" + dst: "{pp}" + is_bind: true + rw: false +}} + "# + ) + }) + .join("\n"); + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{SHARED_MOUNT}", shared_mount) + .replace("{SHARED_DEPENDENCIES}", shared_deps.as_str()) + .replace("{FILE_RESOURCES}", nsjail_extra_mounts.join("\n").as_str()) + .replace( + "{ADDITIONAL_PYTHON_PATHS}", + additional_python_paths_folders.as_str(), + ), + )?; + } else { + reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders); + } let mut cmd_args = vec!["main.yml", "--extra-vars", "@args.json"]; cmd_args.extend(inventories.iter().map(|s| s.as_str())); - let child = { + + let child = if !*DISABLE_NSJAIL { + + let wrapper = format!(r#"set -eou pipefail +{0} "$@" +if [ -f "result" ]; then + cat result > result_nsjail_mount.json +fi +if [ -f "result.json" ]; then + cat result.json > result_nsjail_mount.json +fi +"#, ANSIBLE_PLAYBOOK_PATH.as_str()); + + let file = write_file(job_dir, "wrapper.sh", &wrapper)?; + + file.metadata()?.permissions().set_mode(0o777); + // let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str()); + nsjail_cmd + .current_dir(job_dir) + .env_clear() + // inject PYTHONPATH here - for some reason I had to do it in nsjail conf + .envs(reserved_variables) + .env("PATH", PATH_ENV.as_str()) + .env("TZ", TZ_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("BASE_URL", base_internal_url) + .args( + vec![ + "--config", + "run.config.proto", + "--", + BIN_BASH.as_str(), + "/tmp/wrapper.sh", + ] + .into_iter() + .chain(cmd_args), + ) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await? + } else { let mut ansible_cmd = Command::new(ANSIBLE_PLAYBOOK_PATH.as_str()); ansible_cmd .current_dir(job_dir) @@ -306,6 +406,24 @@ roles_path = ./roles read_and_check_result(job_dir).await } +fn define_nsjail_mount(job_dir: &str, path: &PathBuf) -> anyhow::Result { + Ok(format!( + r#" +mount {{ + src: "{0}/{1}" + dst: "/tmp/{1}" + is_bind: true + rw: false + mandatory: false +}} + "#, + job_dir, + path.strip_prefix(job_dir)? + .to_str() + .ok_or(anyhow!("Invalid path."))? + )) +} + async fn create_file_resources( job_id: &Uuid, w_id: &str, @@ -314,8 +432,9 @@ async fn create_file_resources( r: &AnsibleRequirements, client: &crate::AuthedClient, db: &sqlx::Pool, -) -> error::Result<()> { +) -> error::Result> { let mut logs = String::new(); + let mut nsjail_mounts: Vec = vec![]; for inventory in &r.inventories { let content; @@ -336,9 +455,13 @@ async fn create_file_resources( content = serde_json::from_str(o.get()) .map_err(|e| anyhow!("Failed to parse inventory arg: {}", e))?; + + if content == serde_json::value::Value::Null { + Err(anyhow!("The inventory argument was left empty. If you do not wish to specify an inventory for this script, remove the `inventory:` section from the yaml."))?; + } } - write_file_at_user_defined_location( + let validated_path = write_file_at_user_defined_location( job_dir, &inventory.name, content @@ -349,6 +472,12 @@ async fn create_file_resources( ))?, ) .map_err(|e| anyhow!("Couldn't write inventory: {}", e))?; + + nsjail_mounts.push( + define_nsjail_mount(job_dir, &validated_path) + .map_err(|e| anyhow!("Inventory path (a.k.a. `name`) is invalid: {}", e))?, + ); + logs.push_str(&format!("\nCreated inventory `{}`", inventory.name)); } @@ -360,7 +489,7 @@ async fn create_file_resources( ) .await?; let path = file_res.target_path.clone(); - write_file_at_user_defined_location( + let validated_path = write_file_at_user_defined_location( job_dir, path.as_str(), r.get("content").and_then(|v| v.as_str()).ok_or(anyhow!( @@ -369,6 +498,12 @@ async fn create_file_resources( ))?, ) .map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?; + + nsjail_mounts.push( + define_nsjail_mount(job_dir, &validated_path) + .map_err(|e| anyhow!("File resource path is invalid: {}", e))?, + ); + logs.push_str(&format!( "\nCreated {} from {}", file_res.target_path, file_res.resource_path @@ -376,5 +511,5 @@ async fn create_file_resources( } append_logs(job_id, w_id, logs, db).await; - Ok(()) + Ok(nsjail_mounts) } diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 25d55af57a..412eb6b440 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -12,7 +12,7 @@ use windmill_common::{ use windmill_queue::{append_logs, CanceledBy}; lazy_static::lazy_static! { - static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string()); + pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string()); } const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto"); const NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT: &str = diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index c8538d89ef..ca2f00d4b0 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -14,7 +14,8 @@ use crate::common::build_envs_map; use crate::{ common::{ create_args_and_out_file, get_main_override, get_reserved_variables, handle_child, - parse_npm_config, read_file_content, read_result, start_child_process, write_file_binary, + parse_npm_config, read_file, read_file_content, read_result, start_child_process, + write_file_binary, }, AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, @@ -34,7 +35,7 @@ use windmill_common::variables; use windmill_common::{ error::{self, Result}, get_latest_hash_for_path, - jobs::QueuedJob, + jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, scripts::ScriptLang, worker::{exists_in_cache, get_annotation, save_cache, write_file}, DB, @@ -69,8 +70,7 @@ pub async fn gen_bun_lockfile( raw_deps: Option, npm_mode: bool, ) -> Result> { - let common_bun_proc_envs: HashMap = - get_common_bun_proc_envs(&base_internal_url).await; + let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(None).await; let mut empty_deps = false; @@ -644,8 +644,7 @@ pub async fn prebundle_bun_script( ) .await?; - let common_bun_proc_envs: HashMap = - get_common_bun_proc_envs(&base_internal_url).await; + let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(None).await; generate_bun_bundle( job_dir, @@ -751,6 +750,7 @@ pub async fn handle_bun_job( worker_name: &str, envs: HashMap, shared_mount: &str, + new_args: &mut Option>>, ) -> error::Result> { let mut annotation = windmill_common::worker::get_annotation(inner_content); @@ -779,12 +779,21 @@ pub async fn handle_bun_job( }; let common_bun_proc_envs: HashMap = - get_common_bun_proc_envs(&base_internal_url).await; + get_common_bun_proc_envs(Some(&base_internal_url)).await; if codebase.is_some() { annotation.nodejs_mode = true } - let main_override = get_main_override(job.args.as_ref()); + let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) { + Some(main_override) => { + if main_override == PREPROCESSOR_FAKE_ENTRYPOINT { + (None, true) + } else { + (Some(main_override), false) + } + } + None => (None, false), + }; #[cfg(not(feature = "enterprise"))] if annotation.nodejs_mode || annotation.npm_mode { @@ -939,18 +948,35 @@ pub async fn handle_bun_job( let args = windmill_parser_ts::parse_deno_signature(inner_content, true, main_override.clone())? .args; - let dates = args + + let pre_args = if apply_preprocessor { + Some( + windmill_parser_ts::parse_deno_signature( + inner_content, + true, + Some("preprocessor".to_string()), + )? + .args, + ) + } else { + None + }; + + let dates = pre_args + .as_ref() + .unwrap_or(&args) .iter() - .enumerate() - .filter_map(|(i, x)| { + .filter_map(|x| { if matches!(x.typ, Typ::Datetime) { - Some(i) + Some(x.name.as_str()) } else { None } }) - .map(|x| return format!("args[{x}] = args[{x}] ? new Date(args[{x}]) : undefined")) - .join("\n"); + .map(|x| { + return format!(r#"args["{x}"] = args["{x}"] ? new Date(args["{x}"]) : undefined"#); + }) + .join("\n "); let spread = args.into_iter().map(|x| x.name).join(","); // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str()); @@ -963,22 +989,47 @@ pub async fn handle_bun_job( "./main.ts" }; - let wrapper_content: String = format!( + let preprocessor = if let Some(pre_args) = pre_args { + let pre_spread = pre_args.into_iter().map(|x| x.name).join(","); + format!( + r#"if (Main.preprocessor === undefined || typeof Main.preprocessor !== 'function') {{ + throw new Error("preprocessor function is missing"); + }} + function preArgsObjToArr({{ {pre_spread} }}) {{ + return [ {pre_spread} ]; + }} + args = await Main.preprocessor(...preArgsObjToArr(args)); + const args_json = JSON.stringify(args ?? null, (key, value) => typeof value === 'undefined' ? null : value); + await fs.writeFile('args.json', args_json, {{ encoding: 'utf8' }})"# + ) + } else { + "".to_string() + }; + + let wrapper_content = format!( r#" import * as Main from "{main_import}"; import * as fs from "fs/promises"; -const args = await fs.readFile('args.json', {{ encoding: 'utf8' }}).then(JSON.parse) - .then(({{ {spread} }}) => [ {spread} ]) +let args = await fs.readFile('args.json', {{ encoding: 'utf8' }}).then(JSON.parse); + +function argsObjToArr({{ {spread} }}) {{ + return [ {spread} ]; +}} BigInt.prototype.toJSON = function () {{ return this.toString(); }}; -{dates} async function run() {{ - let res = await Main.{main_name}(...args); + {dates} + {preprocessor} + const argsArr = argsObjToArr(args); + if (Main.{main_name} === undefined || typeof Main.{main_name} !== 'function') {{ + throw new Error("{main_name} function is missing"); + }} + let res = await Main.{main_name}(...argsArr); const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value); await fs.writeFile("result.json", res_json); process.exit(0); @@ -1311,22 +1362,33 @@ try {{ false, ) .await?; + + if apply_preprocessor { + let args = read_file(&format!("{job_dir}/args.json")) + .await + .map_err(|e| { + error::Error::InternalErr(format!( + "error while reading args from preprocessing: {e:#}" + )) + })?; + let args: HashMap> = + serde_json::from_str(args.get()).map_err(|e| { + error::Error::InternalErr(format!( + "error while deserializing args from preprocessing: {e:#}" + )) + })?; + *new_args = Some(args.clone()); + } read_result(job_dir).await } -pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap { +pub async fn get_common_bun_proc_envs(base_internal_url: Option<&str>) -> HashMap { let mut bun_envs: HashMap = HashMap::from([ (String::from("PATH"), PATH_ENV.clone()), (String::from("HOME"), HOME_ENV.clone()), (String::from("TZ"), TZ_ENV.clone()), (String::from("FORCE_COLOR"), "1".to_string()), (String::from("DO_NOT_TRACK"), "1".to_string()), - ( - String::from("BASE_URL"), - base_internal_url - .to_string() - .replace("localhost", "127.0.0.1"), - ), ( String::from("BUN_INSTALL_CACHE_DIR"), BUN_CACHE_DIR.to_string(), @@ -1337,6 +1399,12 @@ pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap = - get_common_bun_proc_envs(&base_internal_url).await; + get_common_bun_proc_envs(Some(&base_internal_url)).await; let mut annotation = windmill_common::worker::get_annotation(inner_content); diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index d60dfd977a..4df77dc21f 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -107,7 +107,7 @@ pub async fn create_args_and_out_file( job_dir: &str, db: &Pool, ) -> Result<(), Error> { - if let Some(args) = &job.args { + if let Some(args) = job.args.as_ref() { if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, db).await? { write_file( job_dir, @@ -380,8 +380,8 @@ pub async fn read_and_check_file(path: &str) -> error::Result> { check_result_too_big(content.len())?; - let raw_value: Box = serde_json::from_str(&content) - .map_err(|e| anyhow!("{} is not valid json: {}", path, e))?; + let raw_value: Box = + serde_json::from_str(&content).map_err(|e| anyhow!("{} is not valid json: {}", path, e))?; Ok(raw_value) } diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 4b139a63d1..146d67be07 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -8,13 +8,15 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ create_args_and_out_file, get_main_override, get_reserved_variables, handle_child, - parse_npm_config, read_result, start_child_process, + parse_npm_config, read_file, read_result, start_child_process, }, AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; use tokio::{fs::File, io::AsyncReadExt, process::Command}; -use windmill_common::{error::Result, worker::write_file, BASE_URL}; +use windmill_common::{ + error::Result, jobs::PREPROCESSOR_FAKE_ENTRYPOINT, worker::write_file, BASE_URL, +}; use windmill_common::{ error::{self}, jobs::QueuedJob, @@ -170,12 +172,22 @@ pub async fn handle_deno_job( base_internal_url: &str, worker_name: &str, envs: HashMap, + new_args: &mut Option>>, ) -> error::Result> { // let mut start = Instant::now(); let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); append_logs(&job.id, &job.workspace_id, logs1, db).await; - let main_override = get_main_override(job.args.as_ref()); + let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) { + Some(main_override) => { + if main_override == PREPROCESSOR_FAKE_ENTRYPOINT { + (None, true) + } else { + (Some(main_override), false) + } + } + None => (None, false), + }; write_file(job_dir, "main.ts", inner_content)?; @@ -184,37 +196,83 @@ pub async fn handle_deno_job( let args = windmill_parser_ts::parse_deno_signature(inner_content, true, main_override.clone())? .args; - let dates = args + + let pre_args = if apply_preprocessor { + Some( + windmill_parser_ts::parse_deno_signature( + inner_content, + true, + Some("preprocessor".to_string()), + )? + .args, + ) + } else { + None + }; + + let dates = pre_args + .as_ref() + .unwrap_or(&args) .iter() - .enumerate() - .filter_map(|(i, x)| { + .filter_map(|x| { if matches!(x.typ, Typ::Datetime) { - Some(i) + Some(x.name.as_str()) } else { None } }) - .map(|x| return format!("args[{x}] = args[{x}] ? new Date(args[{x}]) : undefined")) - .join("\n"); + .map(|x| { + return format!(r#"args["{x}"] = args["{x}"] ? new Date(args["{x}"]) : undefined"#); + }) + .join("\n "); let spread = args.into_iter().map(|x| x.name).join(","); let main_name = main_override.unwrap_or("main".to_string()); // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str()); + let (preprocessor_import, preprocessor) = if let Some(pre_args) = pre_args { + let pre_spread = pre_args.into_iter().map(|x| x.name).join(","); + ( + r#"import { preprocessor } from "./main.ts";"#.to_string(), + format!( + r#"if (preprocessor === undefined || typeof preprocessor !== 'function') {{ + throw new Error("preprocessor function is missing"); + }} + function preArgsObjToArr({{ {pre_spread} }}) {{ + return [ {pre_spread} ]; + }} + args = await preprocessor(...preArgsObjToArr(args)); + const args_json = JSON.stringify(args ?? null, (key, value) => typeof value === 'undefined' ? null : value); + await Deno.writeTextFile("args.json", args_json);"# + ), + ) + } else { + ("".to_string(), "".to_string()) + }; + let wrapper_content: String = format!( r#" import {{ {main_name} }} from "./main.ts"; +{preprocessor_import} -const args = await Deno.readTextFile("args.json") - .then(JSON.parse) - .then(({{ {spread} }}) => [ {spread} ]) +let args = await Deno.readTextFile("args.json") + .then(JSON.parse); + +function argsObjToArr({{ {spread} }}) {{ + return [ {spread} ]; +}} BigInt.prototype.toJSON = function () {{ return this.toString(); }}; -{dates} async function run() {{ - let res: any = await {main_name}(...args); + {dates} + {preprocessor} + const argsArr = argsObjToArr(args); + if ({main_name} === undefined || typeof {main_name} !== 'function') {{ + throw new Error("{main_name} function is missing"); + }} + let res: any = await {main_name}(...argsArr); const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value); await Deno.writeTextFile("result.json", res_json); Deno.exit(0); @@ -346,6 +404,22 @@ try {{ { tracing::error!("failed to remove deno gen tmp cache dir: {}", e); } + if apply_preprocessor { + let args = read_file(&format!("{job_dir}/args.json")) + .await + .map_err(|e| { + error::Error::InternalErr(format!( + "error while reading args from preprocessing: {e:#}" + )) + })?; + let args: HashMap> = + serde_json::from_str(args.get()).map_err(|e| { + error::Error::InternalErr(format!( + "error while deserializing args from preprocessing: {e:#}" + )) + })?; + *new_args = Some(args.clone()); + } read_result(job_dir).await } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 36de523eb3..51584eb037 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -14,7 +14,7 @@ use uuid::Uuid; use windmill_common::ee::{get_license_plan, LicensePlan}; use windmill_common::{ error::{self, Error}, - jobs::QueuedJob, + jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, utils::calculate_hash, worker::{write_file, WORKER_CONFIG}, DB, @@ -56,7 +56,7 @@ use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS; use crate::{ common::{ create_args_and_out_file, get_main_override, get_reserved_variables, handle_child, - read_result, start_child_process, + read_file, read_result, start_child_process, }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, @@ -246,6 +246,7 @@ pub async fn handle_python_job( shared_mount: &str, base_internal_url: &str, envs: HashMap, + new_args: &mut Option>>, ) -> windmill_common::error::Result> { let script_path = crate::common::use_flow_root_path(job.script_path()); let additional_python_paths = handle_python_deps( @@ -281,10 +282,39 @@ pub async fn handle_python_job( transforms, spread, main_name, - ) = prepare_wrapper(job_dir, inner_content, &script_path, job.args.as_ref()).await?; + pre_spread, + ) = prepare_wrapper( + job_dir, + inner_content, + &script_path, + job.args.as_ref(), + false, + ) + .await?; + + let apply_preprocessor = pre_spread.is_some(); create_args_and_out_file(&client, job, job_dir, db).await?; + let preprocessor = if let Some(pre_spread) = pre_spread { + format!( + r#"if inner_script.preprocessor is None or not callable(inner_script.preprocessor): + raise ValueError("preprocessor function is missing") + else: + pre_args = {{}} + {pre_spread} + for k, v in list(pre_args.items()): + if v == '': + del pre_args[k] + kwargs = inner_script.preprocessor(**pre_args) + kwrags_json = res_to_json(kwargs) + with open("args.json", 'w') as f: + f.write(kwrags_json)"# + ) + } else { + "".to_string() + }; + let os_main_override = if let Some(main_override) = main_name.as_ref() { format!("os.environ[\"MAIN_OVERRIDE\"] = \"{main_override}\"\n") } else { @@ -308,10 +338,6 @@ with open("args.json") as f: kwargs = json.load(f, strict=False) args = {{}} {transforms} -{spread} -for k, v in list(args.items()): - if v == '': - del args[k] def to_b_64(v: bytes): import base64 @@ -322,8 +348,7 @@ replace_nan = re.compile(r'(?:\bNaN\b|\\*\\u0000)') result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json") -try: - res = inner_script.{main_override}(**args) +def res_to_json(res): typ = type(res) if typ.__name__ == 'DataFrame': if typ.__module__ == 'pandas.core.frame': @@ -336,7 +361,18 @@ try: for k, v in res.items(): if type(v).__name__ == 'bytes': res[k] = to_b_64(v) - res_json = re.sub(replace_nan, ' null ', json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')) + return re.sub(replace_nan, ' null ', json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')) + +try: + {preprocessor} + {spread} + for k, v in list(args.items()): + if v == '': + del args[k] + if inner_script.{main_override} is None or not callable(inner_script.{main_override}): + raise ValueError("{main_override} function is missing") + res = inner_script.{main_override}(**args) + res_json = res_to_json(res) with open(result_json, 'w') as f: f.write(res_json) except BaseException as e: @@ -451,6 +487,24 @@ mount {{ false, ) .await?; + + if apply_preprocessor { + let args = read_file(&format!("{job_dir}/args.json")) + .await + .map_err(|e| { + error::Error::InternalErr(format!( + "error while reading args from preprocessing: {e:#}" + )) + })?; + let args: HashMap> = + serde_json::from_str(args.get()).map_err(|e| { + error::Error::InternalErr(format!( + "error while deserializing args from preprocessing: {e:#}" + )) + })?; + *new_args = Some(args.clone()); + } + read_result(job_dir).await } @@ -459,6 +513,7 @@ async fn prepare_wrapper( inner_content: &str, script_path: &str, args: Option<&Json>>>, + skip_preprocessor: bool, ) -> error::Result<( &'static str, &'static str, @@ -469,8 +524,18 @@ async fn prepare_wrapper( String, String, Option, + Option, )> { - let main_override = get_main_override(args); + let (main_override, apply_preprocessor) = match get_main_override(args) { + Some(main_override) => { + if !skip_preprocessor && main_override == PREPROCESSOR_FAKE_ENTRYPOINT { + (None, true) + } else { + (Some(main_override), false) + } + } + None => (None, false), + }; let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content); @@ -511,7 +576,20 @@ async fn prepare_wrapper( } let sig = windmill_parser_py::parse_python_signature(inner_content, main_override.clone())?; - let transforms = sig + + let pre_sig = if apply_preprocessor { + Some(windmill_parser_py::parse_python_signature( + inner_content, + Some("preprocessor".to_string()), + )?) + } else { + None + }; + + // transforms should be applied based on the signature of the first function called + let init_sig = pre_sig.as_ref().unwrap_or(&sig); + + let transforms = init_sig .args .iter() .map(|x| match x.typ { @@ -539,7 +617,7 @@ async fn prepare_wrapper( } else { "" }; - let import_base64 = if sig + let import_base64 = if init_sig .args .iter() .any(|x| x.typ == windmill_parser::Typ::Bytes) @@ -548,7 +626,7 @@ async fn prepare_wrapper( } else { "" }; - let import_datetime = if sig + let import_datetime = if init_sig .args .iter() .any(|x| x.typ == windmill_parser::Typ::Datetime) @@ -569,12 +647,38 @@ async fn prepare_wrapper( } else { format!( r#"args["{name}"] = kwargs.get("{name}") -if args["{name}"] is None: - del args["{name}"]"# + if args["{name}"] is None: + del args["{name}"]"# ) } }) - .join("\n") + .join("\n ") + }; + + let pre_spread = if let Some(pre_sig) = pre_sig { + let spread = if pre_sig.star_kwargs { + "pre_args = kwargs".to_string() + } else { + pre_sig + .args + .into_iter() + .map(|x| { + let name = &x.name; + if x.default.is_none() { + format!("pre_args[\"{name}\"] = kwargs.get(\"{name}\")") + } else { + format!( + r#"pre_args["{name}"] = kwargs.get("{name}") + if pre_args["{name}"] is None: + del pre_args["{name}"]"# + ) + } + }) + .join("\n ") + }; + Some(spread) + } else { + None }; let module_dir_dot = dirs.replace("/", ".").replace("-", "_"); @@ -589,6 +693,7 @@ if args["{name}"] is None: transforms, spread, main_override, + pre_spread, )) } @@ -1082,7 +1187,8 @@ pub async fn start_worker( transforms, spread, _, - ) = prepare_wrapper(job_dir, inner_content, script_path, _args.as_ref()).await?; + _, + ) = prepare_wrapper(job_dir, inner_content, script_path, _args.as_ref(), true).await?; { let indented_transforms = transforms @@ -1090,11 +1196,6 @@ pub async fn start_worker( .map(|x| format!(" {}", x)) .collect::>() .join("\n"); - let indented_spread = spread - .lines() - .map(|x| format!(" {}", x)) - .collect::>() - .join("\n"); let wrapper_content: String = format!( r#" @@ -1122,7 +1223,7 @@ for line in sys.stdin: kwargs = json.loads(line, strict=False) args = {{}} {indented_transforms} -{indented_spread} + {spread} for k, v in list(args.items()): if v == '': del args[k] diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index e749f28d3d..9aaa4f2799 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -1,6 +1,6 @@ use serde::Serialize; use sqlx::{types::Json, Pool, Postgres}; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use uuid::Uuid; @@ -53,26 +53,49 @@ pub async fn process_result( cached_res_path: Option, token: String, column_order: Option>, + new_args: Option>>, db: &DB, ) -> error::Result<()> { match result { Ok(r) => { - let job = if let Some(column_order) = column_order { - let mut job_with_column_order = (*job).clone(); - match job_with_column_order.flow_status { - Some(_) => { - tracing::warn!("flow_status was expected to be none"); - } - None => { - job_with_column_order.flow_status = - Some(sqlx::types::Json(to_raw_value(&serde_json::json!({ - "_metadata": { - "column_order": column_order - } - })))); + let job = if column_order.is_some() || new_args.is_some() { + let mut updated_job = (*job).clone(); + if let Some(column_order) = column_order { + match updated_job.flow_status { + Some(_) => { + tracing::warn!("flow_status was expected to be none"); + } + None => { + updated_job.flow_status = + Some(sqlx::types::Json(to_raw_value(&serde_json::json!({ + "_metadata": { + "column_order": column_order + } + })))); + } } } - Arc::new(job_with_column_order) + if let Some(new_args) = new_args { + match updated_job.flow_status { + Some(_) => { + tracing::warn!("flow_status was expected to be none"); + } + None => { + // TODO save original args somewhere + // if let Some(args) = updated_job.args.as_mut() { + // args.0.remove(ENTRYPOINT_OVERRIDE); + // } + updated_job.flow_status = + Some(sqlx::types::Json(to_raw_value(&serde_json::json!({ + "_metadata": { + "preprocessed_args": true + } + })))); + } + } + updated_job.args = Some(Json(new_args)); + } + Arc::new(updated_job) } else { job }; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index ed8e432194..812fd78cd1 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -85,9 +85,9 @@ use rand::Rng; use crate::{ ansible_executor::handle_ansible_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{ build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, - NO_LOGS_AT_ALL, SLOW_LOGS, - }, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{handle_job_error, handle_receive_completed_job, process_result}, rust_executor::handle_rust_job, worker_flow::{ - handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, + NO_LOGS_AT_ALL, SLOW_LOGS, + }, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, handle_job_error, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{handle_receive_completed_job, process_result}, rust_executor::handle_rust_job, worker_flow::{ + handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, Step, }, worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, } @@ -2050,7 +2050,7 @@ async fn handle_queued_job( } #[cfg(any(not(feature = "enterprise"), feature = "sqlx"))] - if job.created_by.starts_with("email-trigger-") { + if job.created_by.starts_with("email-") { let daily_count = sqlx::query!( "SELECT value FROM metrics WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day' ORDER BY created_at DESC LIMIT 1" ).fetch_optional(db).await?.map(|x| serde_json::from_value::(x.value).unwrap_or(1)); @@ -2087,7 +2087,7 @@ async fn handle_queued_job( ) .await?; - r + Some(r) } else { if let Some(parent_job) = job.parent_job { if let Err(e) = sqlx::query_scalar!( @@ -2139,7 +2139,11 @@ async fn handle_queued_job( .await .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e:#}")))? .ok_or_else(|| Error::InternalErr(format!("Expected script_path")))?; - let step = step.unwrap_or(-1); + let step = match step.unwrap() { + Step::Step(i) => i.to_string(), + Step::PreprocessorStep => "preprocessor".to_string(), + Step::FailureStep => "failure".to_string(), + }; Some(format!( "{flow_path}/cache/{version_hash}/{step}/{args_hash}" )) @@ -2239,6 +2243,7 @@ async fn handle_queued_job( append_logs(&job.id, &job.workspace_id, logs, db).await; let mut column_order: Option> = None; + let mut new_args: Option>> = None; let result = match job.job_kind { JobKind::Dependencies => { handle_dependency_job( @@ -2306,6 +2311,7 @@ async fn handle_queued_job( base_internal_url, worker_name, &mut column_order, + &mut new_args, ) .await; *worker_code_execution_metric += metric_timer.elapsed().as_secs_f32(); @@ -2337,6 +2343,7 @@ async fn handle_queued_job( cached_res_path, client.get_token().await, column_order, + new_args, db, ) .await?; @@ -2432,7 +2439,7 @@ pub async fn get_script_content_by_hash( Option, Option, Option>, - Option, + Option ), >( "SELECT content, lock, language, envs, codebase LIKE '%.tar' as codebase FROM script WHERE hash = $1 AND workspace_id = $2", @@ -2473,6 +2480,7 @@ async fn handle_code_execution_job( base_internal_url: &str, worker_name: &str, column_order: &mut Option>, + new_args: &mut Option>>, ) -> error::Result> { let ContentReqLangEnvs { content: inner_content, @@ -2705,6 +2713,7 @@ mount {{ &shared_mount, base_internal_url, envs, + new_args, ) .await } @@ -2721,6 +2730,7 @@ mount {{ base_internal_url, worker_name, envs, + new_args, ) .await } @@ -2739,6 +2749,7 @@ mount {{ worker_name, envs, &shared_mount, + new_args, ) .await } @@ -2838,6 +2849,7 @@ mount {{ db, client, &inner_content, + &shared_mount, base_internal_url, envs, ).await diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 9064cf1579..8c1161cc3e 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -37,7 +37,7 @@ use windmill_common::flow_status::{ use windmill_common::flows::add_virtual_items_if_necessary; use windmill_common::jobs::{ script_hash_to_tag_and_limits, script_path_to_payload, BranchResults, JobPayload, QueuedJob, - RawCode, + RawCode, ENTRYPOINT_OVERRIDE, }; use windmill_common::worker::to_raw_value; use windmill_common::{ @@ -225,14 +225,22 @@ pub async fn update_flow_status_after_job_completion_internal< ))) })?; - let module_index = usize::try_from(old_status.step).ok(); + let module_step = Step::from_i32_and_len(old_status.step, old_status.modules.len()); - let module_status = module_index - .and_then(|i| old_status.modules.get(i)) - .unwrap_or(&old_status.failure_module.module_status); + let module_status = match module_step { + Step::PreprocessorStep => old_status + .preprocessor_module + .as_ref() + .ok_or_else(|| Error::InternalErr(format!("preprocessor module not found")))?, + Step::FailureStep => &old_status.failure_module.module_status, + Step::Step(i) => old_status + .modules + .get(i as usize) + .ok_or_else(|| Error::InternalErr(format!("module {i} not found")))?, + }; // tracing::debug!( - // "UPDATE FLOW STATUS 2: {module_index:#?} {module_status:#?} {old_status:#?} " + // "UPDATE FLOW STATUS 2: {module_step:#?} {module_status:#?} {old_status:#?} " // ); let (is_loop, skip_loop_failures, parallelism) = if matches!( @@ -262,14 +270,14 @@ pub async fn update_flow_status_after_job_completion_internal< let flow_job = get_queued_job(&flow, w_id, db) .await? .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?; - let module = get_module(&flow_job, module_index); + let module = get_module(&flow_job, &module_step); if module.is_some_and(|x| x.is_flow()) { (false, false, false) } else { (true, se, false) } - } else if is_failure_step { + } else if is_failure_step || matches!(module_step, Step::PreprocessorStep) { (false, false, false) } else { let r = sqlx::query_as::<_, SkipIfStopped>( @@ -346,6 +354,28 @@ pub async fn update_flow_status_after_job_completion_internal< let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into(); + if matches!(module_step, Step::PreprocessorStep) { + sqlx::query!( + "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2", + job_id_for_status, + flow + ).execute(&mut tx).await.map_err(|e| { + Error::InternalErr(format!("error while updating args in preprocessing step: {e:#}")) + })?; + + sqlx::query!( + r#"UPDATE completed_job SET args = '{"reason":"PREPROCESSOR_ARGS_ARE_DISCARDED"}'::jsonb WHERE id = $1"#, + job_id_for_status + ) + .execute(&mut tx) + .await + .map_err(|e| { + Error::InternalErr(format!( + "error while deleting args of preprocessing step: {e:#}" + )) + })?; + } + let (inc_step_counter, new_status) = match module_status { FlowStatusModule::InProgress { iterator, @@ -713,6 +743,21 @@ pub async fn update_flow_status_after_job_completion_internal< "error while setting flow status in failure step: {e:#}" )) })?; + } else if matches!(module_step, Step::PreprocessorStep) { + sqlx::query!( + "UPDATE queue + SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1) + WHERE id = $2", + json!(new_status), + flow + ) + .execute(&mut tx) + .await + .map_err(|e| { + Error::InternalErr(format!( + "error while setting flow status in preprocessing step: {e:#}" + )) + })?; } else { sqlx::query!( "UPDATE queue @@ -824,8 +869,7 @@ pub async fn update_flow_status_after_job_completion_internal< .unwrap_or_else(|| "none".to_string()); tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status"); - let module = get_module(&flow_job, module_index); - + let module = get_module(&flow_job, &module_step); // tracing::error!( // "UPDATE FLOW STATUS 3: {module:#?} {unrecoverable} {} {is_last_step} {success} {skip_error_handler} is_failure_step {is_failure_step}", flow_job.canceled // ); @@ -1115,17 +1159,13 @@ async fn retrieve_flow_jobs_results( Ok(to_raw_value(&results)) } -fn get_module(flow_job: &QueuedJob, module_index: Option) -> Option { +fn get_module(flow_job: &QueuedJob, module_step: &Step) -> Option { let raw_flow = flow_job.parse_raw_flow(); if let Some(raw_flow) = raw_flow { - if let Some(i) = module_index { - if let Some(module) = raw_flow.modules.get(i) { - Some(module.clone()) - } else { - raw_flow.failure_module.map(|x| *x.clone()) - } - } else { - None + match module_step { + Step::PreprocessorStep => raw_flow.preprocessor_module.map(|x| *x.clone()), + Step::Step(i) => raw_flow.modules.get(*i).map(|x| x.clone()), + Step::FailureStep => raw_flow.failure_module.map(|x| *x.clone()), } } else { None @@ -1293,56 +1333,86 @@ pub async fn update_flow_status_in_progress( w_id: &str, flow: Uuid, job_in_progress: Uuid, -) -> error::Result> { +) -> error::Result { let step = get_step_of_flow_status(db, flow).await?; - if let Step::Step(step) = step { - sqlx::query(&format!( - "UPDATE queue - SET flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {step}, job}}', $1), '{{modules, {step}, type}}', $2) - WHERE id = $3 AND workspace_id = $4", - )) - .bind(json!(job_in_progress.to_string())) - .bind(json!("InProgress")) - .bind(flow) - .bind(w_id) - .execute(db) - .await?; - Ok(Some(step)) - } else { - sqlx::query(&format!( - "UPDATE queue - SET flow_status = jsonb_set(jsonb_set(flow_status, '{{failure_module, job}}', $1), '{{failure_module, type}}', $2) - WHERE id = $3 AND workspace_id = $4", - )) - .bind(json!(job_in_progress.to_string())) - .bind(json!("InProgress")) - .bind(flow) - .bind(w_id) - .execute(db) - .await?; - Ok(None) + match step { + Step::Step(step) => { + sqlx::query(&format!( + "UPDATE queue + SET flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {step}, job}}', $1), '{{modules, {step}, type}}', $2) + WHERE id = $3 AND workspace_id = $4", + )) + .bind(json!(job_in_progress.to_string())) + .bind(json!("InProgress")) + .bind(flow) + .bind(w_id) + .execute(db) + .await?; + } + Step::PreprocessorStep => { + sqlx::query(&format!( + "UPDATE queue + SET flow_status = jsonb_set(jsonb_set(flow_status, '{{preprocessor_module, job}}', $1), '{{preprocessor_module, type}}', $2) + WHERE id = $3 AND workspace_id = $4", + )) + .bind(json!(job_in_progress.to_string())) + .bind(json!("InProgress")) + .bind(flow) + .bind(w_id) + .execute(db) + .await?; + } + Step::FailureStep => { + sqlx::query(&format!( + "UPDATE queue + SET flow_status = jsonb_set(jsonb_set(flow_status, '{{failure_module, job}}', $1), '{{failure_module, type}}', $2) + WHERE id = $3 AND workspace_id = $4", + )) + .bind(json!(job_in_progress.to_string())) + .bind(json!("InProgress")) + .bind(flow) + .bind(w_id) + .execute(db) + .await?; + } + } + + Ok(step) +} + +#[derive(Debug)] +pub enum Step { + Step(usize), + PreprocessorStep, + FailureStep, +} + +impl Step { + fn from_i32_and_len(step: i32, len: usize) -> Self { + if step < 0 { + Step::PreprocessorStep + } else if (step as usize) < len { + Step::Step(step as usize) + } else { + Step::FailureStep + } } } -pub enum Step { - Step(i32), - FailureStep, -} #[instrument(level = "trace", skip_all)] pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result { let r = sqlx::query!( - "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1", + "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1", id ) .fetch_one(db) .await .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e:#}")))?; - if r.step < r.len { - Ok(Step::Step(r.step.ok_or_else(|| { - Error::InternalErr("step is null".to_string()) - })?)) + + if let Some(step) = r.step { + Ok(Step::from_i32_and_len(step, r.len.unwrap_or(0) as usize)) } else { - Ok(Step::FailureStep) + Err(Error::InternalErr("step is null".to_string())) } } @@ -1539,14 +1609,20 @@ async fn push_next_flow_job .unwrap_or_else(|| "none".to_string()); tracing::info!(id = %flow_job.id, root_id = %job_root, "pushing next flow job"); - let mut i = usize::try_from(status.step) - .with_context(|| format!("invalid module index {}", status.step))?; + let mut step = Step::from_i32_and_len(status.step, flow.modules.len()); - let mut status_module: FlowStatusModule = status - .modules - .get(i) - .cloned() - .unwrap_or_else(|| status.failure_module.module_status.clone()); + let mut status_module = match step { + Step::Step(i) => status + .modules + .get(i) + .cloned() + .unwrap_or_else(|| status.failure_module.module_status.clone()), + Step::PreprocessorStep => status + .preprocessor_module + .clone() + .unwrap_or_else(|| status.failure_module.module_status.clone()), + Step::FailureStep => status.failure_module.module_status.clone(), + }; let fj: mappable_rc::Marc = flow_job.clone().into(); let arc_flow_job_args: Marc>> = Marc::map(fj, |x| { @@ -1584,7 +1660,7 @@ async fn push_next_flow_job return Ok(()); } - if i == 0 { + if matches!(step, Step::Step(0)) { if !flow_job.is_flow_step && flow_job.schedule_path.is_some() { let no_flow_overlap = sqlx::query_scalar!( "SELECT no_flow_overlap FROM schedule WHERE path = $1 AND workspace_id = $2", @@ -1672,7 +1748,7 @@ async fn push_next_flow_job let arc_last_job_result = if status_module.is_failure() { // if job is being retried, pass the result of its previous failure last_job_result.unwrap_or_else(|| Arc::new(to_raw_value(&json!("{}")))) - } else if i == 0 { + } else if matches!(step, Step::Step(0)) || matches!(step, Step::PreprocessorStep) { // if it's the first job executed in the flow, pass the flow args Arc::new(to_raw_value(&flow_job.args)) } else { @@ -1929,17 +2005,25 @@ async fn push_next_flow_job } } - let mut module: &FlowModule = flow - .modules - .get(i) - .or_else(|| flow.failure_module.as_deref()) - .with_context(|| format!("no module at index {}", status.step))?; + let mut module = match step { + Step::Step(i) => flow + .modules + .get(i) + .with_context(|| format!("no module at index {}", i))?, + Step::PreprocessorStep => flow + .preprocessor_module + .as_ref() + .with_context(|| format!("no preprocessor module"))?, + Step::FailureStep => flow + .failure_module + .as_deref() + .with_context(|| format!("no failure module"))?, + }; let current_id = &module.id; - let mut previous_id = if i >= 1 { - flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap() - } else { - String::new() + let mut previous_id = match step { + Step::Step(i) if i >= 1 => flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap(), + _ => String::new(), }; // calculate sleep if any @@ -1952,10 +2036,13 @@ async fn push_next_flow_job ) { None } else { - let sleep_input_transform = i - .checked_sub(1) - .and_then(|i| flow.modules.get(i)) - .and_then(|m| m.sleep.clone()); + let sleep_input_transform = if let Step::Step(i) = step { + i.checked_sub(1) + .and_then(|i| flow.modules.get(i)) + .and_then(|m| m.sleep.clone()) + } else { + None + }; if let Some(it) = sleep_input_transform { let json_value = match it { @@ -2054,7 +2141,7 @@ async fn push_next_flow_job * The failure module may also run again if it fails and the retry feature is used. * In that case, `i` will index past `flow.modules`. The above should handle that and * re-run the failure module. */ - i = flow.modules.len(); + step = Step::FailureStep; previous_id = current_id.clone(); module = flow .failure_module @@ -2121,6 +2208,13 @@ async fn push_next_flow_job } else { Ok(Marc::new(HashMap::new())) } + } else if matches!(step, Step::PreprocessorStep) { + let mut hm = (*arc_flow_job_args).clone(); + hm.insert( + ENTRYPOINT_OVERRIDE.to_string(), + to_raw_value(&"preprocessor"), + ); + Ok(Marc::new(hm)) } else { match &module.get_value() { Ok( @@ -2522,6 +2616,7 @@ async fn push_next_flow_job tracing::debug!(id = %flow_job.id, root_id = %job_root, "updated parallel monitor lock for {uuid}"); } } + let first_uuid = uuids[0]; let new_status = match next_status { NextStatus::NextLoopIteration { @@ -2610,36 +2705,52 @@ async fn push_next_flow_job } }; - tracing::debug!("STATUS STEP: {:?} {i} {:#?}", status.step, new_status); + tracing::debug!("STATUS STEP: {:?} {step:?} {:#?}", status.step, new_status); - if i >= flow.modules.len() { - sqlx::query!( - "UPDATE queue - SET flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2) - WHERE id = $3", - json!(FlowStatusModuleWParent { - parent_module: Some(current_id.clone()), - module_status: new_status - }), - json!(i), - flow_job.id - ) - .execute(&mut tx) - .await?; - } else { - sqlx::query!( - "UPDATE queue - SET flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3) - WHERE id = $4", - i as i32, - json!(new_status), - json!(i), - flow_job.id - ) - .execute(&mut tx) - .await?; + match step { + Step::FailureStep => { + sqlx::query!( + "UPDATE queue + SET flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2) + WHERE id = $3", + json!(FlowStatusModuleWParent { + parent_module: Some(current_id.clone()), + module_status: new_status + }), + json!(flow.modules.len()), + flow_job.id + ) + .execute(&mut tx) + .await?; + } + Step::PreprocessorStep => { + sqlx::query!( + "UPDATE queue + SET flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), ARRAY['step'], $2) + WHERE id = $3", + json!(new_status), + json!(-1), + flow_job.id + ) + .execute(&mut tx) + .await?; + } + Step::Step(i) => { + sqlx::query!( + "UPDATE queue + SET flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3) + WHERE id = $4", + i as i32, + json!(new_status), + json!(i), + flow_job.id + ) + .execute(&mut tx) + .await?; + } }; potentially_crash_for_testing(); @@ -2882,9 +2993,17 @@ async fn compute_next_flow_transform( concurrency_time_window_s, .. } => { - let path = path - .clone() - .or_else(|| Some(format!("{}/step-{}", flow_job.script_path(), status.step))); + let path = path.clone().or_else(|| { + if status + .preprocessor_module + .as_ref() + .is_some_and(|x| x.id() == module.id) + { + Some(format!("{}/preprocessor", flow_job.script_path())) + } else { + Some(format!("{}/step-{}", flow_job.script_path(), status.step)) + } + }); let payload = raw_script_to_payload( path, content, @@ -3011,6 +3130,7 @@ async fn compute_next_flow_transform( priority: None, early_return: None, concurrency_key: None, + preprocessor_module: None, }; ContinuePayload::ForloopJobs { n: itered.len(), @@ -3108,6 +3228,7 @@ async fn compute_next_flow_transform( priority: None, early_return: None, concurrency_key: None, + preprocessor_module: None, }, path: Some(format!( "{}/branchone-{}", @@ -3160,6 +3281,7 @@ async fn compute_next_flow_transform( priority: None, early_return: None, concurrency_key: None, + preprocessor_module: None, }, path: Some(format!( "{}/branchall-{}", @@ -3234,6 +3356,7 @@ async fn compute_next_flow_transform( priority: None, early_return: None, concurrency_key: None, + preprocessor_module: None, }, path: Some(format!( "{}/branchall-{}", @@ -3311,6 +3434,7 @@ async fn next_loop_iteration( priority: None, early_return: None, concurrency_key: None, + preprocessor_module: None, }, path: inner_path, restarted_from: None, @@ -3548,7 +3672,11 @@ fn raw_script_to_payload( } fn flow_to_payload(path: &str, delete_after_use: &bool) -> JobPayloadWithTag { - let payload = JobPayload::Flow { path: path.to_string(), dedicated_worker: None }; + let payload = JobPayload::Flow { + path: path.to_string(), + dedicated_worker: None, + apply_preprocessor: false, + }; JobPayloadWithTag { payload, tag: None, delete_after_use: *delete_after_use, timeout: None } } @@ -3562,7 +3690,7 @@ async fn script_to_payload( ) -> Result { let (payload, tag, delete_after_use, script_timeout) = if script_hash.is_none() { let (jp, tag, delete_after_use, script_timeout) = - script_path_to_payload(script_path, db, &flow_job.workspace_id).await?; + script_path_to_payload(script_path, db, &flow_job.workspace_id, Some(true)).await?; ( jp, tag_override.to_owned().or(tag), @@ -3595,6 +3723,7 @@ async fn script_to_payload( language, dedicated_worker, priority, + apply_preprocessor: false, }, tag_override.to_owned().or(tag), delete_after_use, diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index fb32b3519a..e3171ae88b 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.397.1"; +export const VERSION = "v1.398.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 402584a5c6..40c2a02bfd 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -56,7 +56,7 @@ export { // } // }); -export const VERSION = "1.397.1"; +export const VERSION = "1.398.1"; const command = new Command() .name("wmill") diff --git a/cli/metadata.ts b/cli/metadata.ts index ae145504fe..0753496055 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -299,13 +299,15 @@ export async function updateScriptSchema( ): Promise { // infer schema from script content and update it inplace await instantiateWasm(); - const newSchema = inferSchema( + const result = inferSchema( language, scriptContent, metadataContent.schema, path ); - metadataContent.schema = newSchema; + metadataContent.schema = result.schema; + metadataContent.has_preprocessor = result.has_preprocessor; + metadataContent.no_main_func = result.no_main_func; } async function updateScriptLock( @@ -489,7 +491,11 @@ export function inferSchema( `Script ${path} invalid, it cannot be parsed to infer schema.` ) ); - return defaultScriptMetadata().schema; + return { + schema: defaultScriptMetadata().schema, + has_preprocessor: false, + no_main_func: false, + }; } currentSchema.required = []; @@ -515,7 +521,11 @@ export function inferSchema( } } - return currentSchema; + return { + schema: currentSchema, + has_preprocessor: inferedSchema.has_preprocessor, + no_main_func: inferedSchema.no_main_func, + }; } function sortObject(obj: any): any { diff --git a/cli/script.ts b/cli/script.ts index 55b0055e85..6d0b438c7b 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -97,7 +97,6 @@ async function push(opts: PushOptions, filePath: string) { } export async function findResourceFile(path: string) { - const splitPath = path.split("."); const contentBasePathJSON = splitPath[0] + "." + splitPath[1] + ".json"; @@ -124,9 +123,7 @@ export async function findResourceFile(path: string) { ); } if (validCandidates.length < 1) { - throw new Error( - `No resource matching file resource: ${path}.` - ); + throw new Error(`No resource matching file resource: ${path}.`); } return validCandidates[0]; } @@ -306,6 +303,7 @@ export async function handleFile( restart_unless_cancelled: typed?.restart_unless_cancelled, visible_to_runner_only: typed?.visible_to_runner_only, no_main_func: typed?.no_main_func, + has_preprocessor: typed?.has_preprocessor, priority: typed?.priority, concurrency_key: typed?.concurrency_key, //@ts-ignore @@ -339,6 +337,8 @@ export async function handleFile( Boolean(typed.visible_to_runner_only) == Boolean(remote.visible_to_runner_only) && Boolean(typed.no_main_func) == Boolean(remote.no_main_func) && + Boolean(typed.has_preprocessor) == + Boolean(remote.has_preprocessor) && typed.priority == Boolean(remote.priority) && typed.timeout == remote.timeout && //@ts-ignore diff --git a/cli/sync.ts b/cli/sync.ts index 15cb9da08b..d770029c45 100644 --- a/cli/sync.ts +++ b/cli/sync.ts @@ -42,11 +42,8 @@ import { generateFlowLockInternal, generateScriptMetadataInternal, } from "./metadata.ts"; -<<<<<<< HEAD -import { pushResource } from "./resource.ts"; -======= import { FlowModule, OpenFlow, RawScript } from "./gen/types.gen.ts"; ->>>>>>> origin/main +import { pushResource } from "./resource.ts"; type DynFSElement = { isDirectory: boolean; @@ -342,13 +339,15 @@ function ZipFSElement( zip: JSZip, useYaml: boolean, defaultTs: "bun" | "deno", - resourceTypeToFormatExtension: Map, + resourceTypeToFormatExtension: Record ): DynFSElement { async function _internal_file( p: string, f: JSZip.JSZipObject ): Promise { - const kind: "flow" | "app" | "script" | "resource" | "other" = p.endsWith("flow.json") + const kind: "flow" | "app" | "script" | "resource" | "other" = p.endsWith( + "flow.json" + ) ? "flow" : p.endsWith("app.json") ? "app" @@ -457,16 +456,18 @@ function ZipFSElement( : JSON.stringify(parsed, null, 2); } - if (kind == 'resource') { + if (kind == "resource") { const content = await f.async("text"); const parsed = JSON.parse(content); - const formatExtension = resourceTypeToFormatExtension[parsed["resource_type"]] + const formatExtension = + resourceTypeToFormatExtension[parsed["resource_type"]]; if (formatExtension) { - parsed["value"]["content"] = "!inline " - + removeSuffix(p.replaceAll(SEP, "/"), ".resource.json") - + ".resource.file." + formatExtension - + parsed["value"]["content"] = + "!inline " + + removeSuffix(p.replaceAll(SEP, "/"), ".resource.json") + + ".resource.file." + + formatExtension; } return useYaml ? yamlStringify(parsed, yamlOptions) @@ -498,13 +499,17 @@ function ZipFSElement( if (kind == "resource") { const content = await f.async("text"); const parsed = JSON.parse(content); - const formatExtension = resourceTypeToFormatExtension[parsed["resource_type"]] + const formatExtension = + resourceTypeToFormatExtension[parsed["resource_type"]]; if (formatExtension) { const fileContent: string = parsed["value"]["content"]; r.push({ isDirectory: false, - path: removeSuffix(finalPath, ".resource.json") + ".resource.file." + formatExtension, + path: + removeSuffix(finalPath, ".resource.json") + + ".resource.file." + + formatExtension, async *getChildren() {}, // deno-lint-ignore require-await async getContentText() { @@ -895,8 +900,14 @@ export async function pull(opts: GlobalOptions & SyncOptions) { ) ); - const resourceTypeToFormatExtension = - await wmill.fileResourceTypeToFileExtMap({ workspace: workspace.workspaceId }) + let resourceTypeToFormatExtension: Record = {}; + try { + resourceTypeToFormatExtension = (await wmill.fileResourceTypeToFileExtMap({ + workspace: workspace.workspaceId, + })) as Record; + } catch { + // ignore + } const remote = ZipFSElement( (await downloadZip( workspace, @@ -913,7 +924,7 @@ export async function pull(opts: GlobalOptions & SyncOptions) { ))!, !opts.json, opts.defaultTs ?? "bun", - resourceTypeToFormatExtension, + resourceTypeToFormatExtension ); const local = !opts.stateful ? await FSFSElement(Deno.cwd(), codebases) @@ -1188,8 +1199,14 @@ export async function push(opts: GlobalOptions & SyncOptions) { "Computing the files to update on the remote to match local (taking wmill.yaml includes/excludes into account)" ) ); - const resourceTypeToFormatExtension = - await wmill.fileResourceTypeToFileExtMap({ workspace: workspace.workspaceId }) + let resourceTypeToFormatExtension: Record = {}; + try { + resourceTypeToFormatExtension = (await wmill.fileResourceTypeToFileExtMap({ + workspace: workspace.workspaceId, + })) as Record; + } catch { + // ignore + } const remote = ZipFSElement( (await downloadZip( workspace, @@ -1206,7 +1223,7 @@ export async function push(opts: GlobalOptions & SyncOptions) { ))!, !opts.json, opts.defaultTs ?? "bun", - resourceTypeToFormatExtension, + resourceTypeToFormatExtension ); const local = await FSFSElement(path.join(Deno.cwd(), ""), codebases); @@ -1291,24 +1308,26 @@ export async function push(opts: GlobalOptions & SyncOptions) { } if (isFileResource(change.path)) { - const resourceFilePath = await findResourceFile(change.path) + const resourceFilePath = await findResourceFile(change.path); if (!alreadySynced.includes(resourceFilePath)) { alreadySynced.push(resourceFilePath); - const newObj = parseFromPath(resourceFilePath, await Deno.readTextFile(resourceFilePath)); + const newObj = parseFromPath( + resourceFilePath, + await Deno.readTextFile(resourceFilePath) + ); await pushResource( workspace.workspaceId, resourceFilePath, undefined, - newObj, + newObj ); if (opts.stateful && stateExists) { await Deno.writeTextFile(stateTarget, change.after); } continue; } - } const oldObj = parseFromPath(change.path, change.before); const newObj = parseFromPath(change.path, change.after); @@ -1320,7 +1339,7 @@ export async function push(opts: GlobalOptions & SyncOptions) { newObj, opts.plainSecrets ?? false, alreadySynced, - opts.message, + opts.message ); if (opts.stateful && stateExists) { @@ -1359,7 +1378,7 @@ export async function push(opts: GlobalOptions & SyncOptions) { obj, opts.plainSecrets ?? false, [], - opts.message, + opts.message ); if (opts.stateful && stateExists) { diff --git a/cli/utils.ts b/cli/utils.ts index f0927baba4..5f2de77f6b 100644 --- a/cli/utils.ts +++ b/cli/utils.ts @@ -3,6 +3,7 @@ // @ts-nocheck This file is copied from a JS project, so it's not type-safe. import { log, encodeHex, SEP } from "./deps.ts"; +import crypto from "node:crypto"; export function deepEqual(a: T, b: T): boolean { if (a === b) return true; @@ -132,6 +133,9 @@ export function sleep(ms: number) { export function isFileResource(path: string): boolean { const splitPath = path.split("."); - return splitPath.length >= 4 && splitPath[1] == "resource" && splitPath[2] == "file"; + return ( + splitPath.length >= 4 && + splitPath[1] == "resource" && + splitPath[2] == "file" + ); } - diff --git a/cli/wasm/windmill_parser_wasm.generated.d.ts b/cli/wasm/windmill_parser_wasm.generated.d.ts index 10824a1c62..547c600062 100644 --- a/cli/wasm/windmill_parser_wasm.generated.d.ts +++ b/cli/wasm/windmill_parser_wasm.generated.d.ts @@ -19,7 +19,8 @@ export interface InstantiateResult { parse_db_resource: typeof parse_db_resource; parse_graphql: typeof parse_graphql; parse_php: typeof parse_php; - parse_rust: typeof parse_rust + parse_rust: typeof parse_rust; + parse_ansible: typeof parse_ansible }; } @@ -46,9 +47,10 @@ export function instantiateWithInstance(opts?: InstantiateOptions): Promise -// source-hash: d36b5583d8f6223c130a246a8713b8621d55b990 +// source-hash: 6da62ad21f79c5b8fe442f1de939e367943e2665 let wasm; const heap = new Array(128).fill(undefined); @@ -15,6 +15,20 @@ function getObject(idx) { return heap[idx]; } +let heap_next = heap.length; + +function dropObject(idx) { + if (idx < 132) return; + heap[idx] = heap_next; + heap_next = idx; +} + +function takeObject(idx) { + const ret = getObject(idx); + dropObject(idx); + return ret; +} + let WASM_VECTOR_LEN = 0; let cachedUint8Memory0 = null; @@ -89,38 +103,6 @@ function getInt32Memory0() { return cachedInt32Memory0; } -let heap_next = heap.length; - -function dropObject(idx) { - if (idx < 132) return; - heap[idx] = heap_next; - heap_next = idx; -} - -function takeObject(idx) { - const ret = getObject(idx); - dropObject(idx); - return ret; -} - -let cachedFloat64Memory0 = null; - -function getFloat64Memory0() { - if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) { - cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer); - } - return cachedFloat64Memory0; -} - -function addHeapObject(obj) { - if (heap_next === heap.length) heap.push(heap.length + 1); - const idx = heap_next; - heap_next = heap[idx]; - - heap[idx] = obj; - return idx; -} - const cachedTextDecoder = typeof TextDecoder !== "undefined" ? new TextDecoder("utf-8", { ignoreBOM: true, fatal: true }) : { @@ -136,6 +118,24 @@ function getStringFromWasm0(ptr, len) { return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len)); } +function addHeapObject(obj) { + if (heap_next === heap.length) heap.push(heap.length + 1); + const idx = heap_next; + heap_next = heap[idx]; + + heap[idx] = obj; + return idx; +} + +let cachedFloat64Memory0 = null; + +function getFloat64Memory0() { + if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) { + cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer); + } + return cachedFloat64Memory0; +} + let cachedBigInt64Memory0 = null; function getBigInt64Memory0() { @@ -213,11 +213,12 @@ function debugString(val) { } /** * @param {string} code + * @param {string | undefined} [main_override] * @returns {string} */ -export function parse_deno(code) { - let deferred2_0; - let deferred2_1; +export function parse_deno(code, main_override) { + let deferred3_0; + let deferred3_1; try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); const ptr0 = passStringToWasm0( @@ -226,15 +227,23 @@ export function parse_deno(code) { wasm.__wbindgen_realloc, ); const len0 = WASM_VECTOR_LEN; - wasm.parse_deno(retptr, ptr0, len0); + var ptr1 = isLikeNone(main_override) + ? 0 + : passStringToWasm0( + main_override, + wasm.__wbindgen_malloc, + wasm.__wbindgen_realloc, + ); + var len1 = WASM_VECTOR_LEN; + wasm.parse_deno(retptr, ptr0, len0, ptr1, len1); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; - deferred2_0 = r0; - deferred2_1 = r1; + deferred3_0 = r0; + deferred3_1 = r1; return getStringFromWasm0(r0, r1); } finally { wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + wasm.__wbindgen_free(deferred3_0, deferred3_1, 1); } } @@ -375,11 +384,12 @@ export function parse_go(code) { /** * @param {string} code + * @param {string | undefined} [main_override] * @returns {string} */ -export function parse_python(code) { - let deferred2_0; - let deferred2_1; +export function parse_python(code, main_override) { + let deferred3_0; + let deferred3_1; try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); const ptr0 = passStringToWasm0( @@ -388,15 +398,23 @@ export function parse_python(code) { wasm.__wbindgen_realloc, ); const len0 = WASM_VECTOR_LEN; - wasm.parse_python(retptr, ptr0, len0); + var ptr1 = isLikeNone(main_override) + ? 0 + : passStringToWasm0( + main_override, + wasm.__wbindgen_malloc, + wasm.__wbindgen_realloc, + ); + var len1 = WASM_VECTOR_LEN; + wasm.parse_python(retptr, ptr0, len0, ptr1, len1); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; - deferred2_0 = r0; - deferred2_1 = r1; + deferred3_0 = r0; + deferred3_1 = r1; return getStringFromWasm0(r0, r1); } finally { wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + wasm.__wbindgen_free(deferred3_0, deferred3_1, 1); } } @@ -643,6 +661,33 @@ export function parse_rust(code) { } } +/** + * @param {string} code + * @returns {string} + */ +export function parse_ansible(code) { + let deferred2_0; + let deferred2_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0( + code, + wasm.__wbindgen_malloc, + wasm.__wbindgen_realloc, + ); + const len0 = WASM_VECTOR_LEN; + wasm.parse_ansible(retptr, ptr0, len0); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + deferred2_0 = r0; + deferred2_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + function handleError(f, args) { try { return f.apply(this, args); @@ -653,6 +698,9 @@ function handleError(f, args) { const imports = { __wbindgen_placeholder__: { + __wbindgen_object_drop_ref: function (arg0) { + takeObject(arg0); + }, __wbindgen_string_get: function (arg0, arg1) { const obj = getObject(arg1); const ret = typeof obj === "string" ? obj : undefined; @@ -667,8 +715,9 @@ const imports = { getInt32Memory0()[arg0 / 4 + 1] = len1; getInt32Memory0()[arg0 / 4 + 0] = ptr1; }, - __wbindgen_object_drop_ref: function (arg0) { - takeObject(arg0); + __wbindgen_error_new: function (arg0, arg1) { + const ret = new Error(getStringFromWasm0(arg0, arg1)); + return addHeapObject(ret); }, __wbindgen_boolean_get: function (arg0) { const v = getObject(arg0); @@ -706,11 +755,7 @@ const imports = { const ret = BigInt.asUintN(64, arg0); return addHeapObject(ret); }, - __wbindgen_error_new: function (arg0, arg1) { - const ret = new Error(getStringFromWasm0(arg0, arg1)); - return addHeapObject(ret); - }, - __wbg_eval_c227cc6614cd76e5: function (arg0, arg1) { + __wbg_eval_b7405fdd08ddef3d: function (arg0, arg1) { const ret = eval(getStringFromWasm0(arg0, arg1)); return addHeapObject(ret); }, @@ -993,6 +1038,7 @@ function getWasmInstanceExports() { parse_graphql, parse_php, parse_rust, + parse_ansible, }; } diff --git a/cli/wasm/windmill_parser_wasm_bg.wasm b/cli/wasm/windmill_parser_wasm_bg.wasm index de163879c8..ef578c3cf2 100644 Binary files a/cli/wasm/windmill_parser_wasm_bg.wasm and b/cli/wasm/windmill_parser_wasm_bg.wasm differ diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index ea8f9d8c50..b94bdec85e 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -1,7 +1,9 @@ -ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm +ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm FROM ${PYTHON_IMAGE} +RUN pip install --upgrade pip==24.2 + ARG APP=/usr/src/app RUN apt-get update \ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index ea8f9d8c50..b94bdec85e 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -1,7 +1,9 @@ -ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm +ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm FROM ${PYTHON_IMAGE} +RUN pip install --upgrade pip==24.2 + ARG APP=/usr/src/app RUN apt-get update \ diff --git a/docker/RHEL9/Dockerfile b/docker/RHEL9/Dockerfile index d06ca7eed6..3a9ed24bb6 100644 --- a/docker/RHEL9/Dockerfile +++ b/docker/RHEL9/Dockerfile @@ -1,6 +1,6 @@ ARG DEBIAN_IMAGE=debian:bookworm-slim ARG RUST_IMAGE=registry.access.redhat.com/ubi9/ubi:latest -ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm +ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm FROM ${RUST_IMAGE} AS rust_base diff --git a/frontend/package-lock.json b/frontend/package-lock.json index b732e98336..6a225b2de4 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.397.1", + "version": "1.398.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.397.1", + "version": "1.398.1", "license": "AGPL-3.0", "dependencies": { "@aws-crypto/sha256-js": "^4.0.0", @@ -66,12 +66,12 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.0.8", "vscode-ws-jsonrpc": "~3.3.2", - "windmill-parser-wasm-go": "^1.382.2", - "windmill-parser-wasm-php": "^1.382.2", - "windmill-parser-wasm-py": "^1.382.2", - "windmill-parser-wasm-regex": "^1.382.2", - "windmill-parser-wasm-rust": "^1.387.0", - "windmill-parser-wasm-ts": "^1.382.2", + "windmill-parser-wasm-go": "^1.397.2", + "windmill-parser-wasm-php": "^1.397.2", + "windmill-parser-wasm-py": "^1.397.2", + "windmill-parser-wasm-regex": "^1.397.2", + "windmill-parser-wasm-rust": "^1.397.2", + "windmill-parser-wasm-ts": "^1.397.2", "windmill-parser-wasm-yaml": "^1.396.1", "windmill-sql-datatype-parser-wasm": "^1.318.0", "y-monaco": "^0.1.4", @@ -13495,34 +13495,34 @@ } }, "node_modules/windmill-parser-wasm-go": { - "version": "1.385.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.385.0.tgz", - "integrity": "sha512-au+i0TDbhnklDrpGTYSXVwqaY4DSKC17anHFLjeG8oHJ9xlc/kNG+2PASs2sqc5m7e6G6D78LDt/5FYs1b9b+g==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.397.2.tgz", + "integrity": "sha512-je5X7nsbudwlZL6QLpA/s1fwTip0Xmcc8c8eciJjK9IY6JZd2s8IRmtgVb3MB1ppQ4poCzypeqpYcvlL4MOGlg==" }, "node_modules/windmill-parser-wasm-php": { - "version": "1.385.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-php/-/windmill-parser-wasm-php-1.385.0.tgz", - "integrity": "sha512-3x6fFy19BXNfLhzctFEh5fozrKBoWUla/JsZpYaKPw7AKbQ/aP1o+SKZDSrXStaNDHfipdsPYLZ0MIxjZY6p1Q==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-php/-/windmill-parser-wasm-php-1.397.2.tgz", + "integrity": "sha512-+ys3e84UojguDxSJurMT/T2m0XGYUyE59+TbIUteWjxyMnSoynC3GhYvWU3FX0yojwiLOdDaUZkGlkNy8Fl3+w==" }, "node_modules/windmill-parser-wasm-py": { - "version": "1.385.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.385.0.tgz", - "integrity": "sha512-v+6hZNGBp+8KMoAkWIlt5Eu+0n1mNf7vrk/bHr0X2s78cM4JhTXZ24eR7xx69eesQpO7X87pEn/HJHIUHC0kAQ==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.397.2.tgz", + "integrity": "sha512-gnqQZTRmQi2s3ok9IyixTHGK3MOfJpW2+nnH4W8vXGj1VQMGoXScaZA1W8Nt6/lFy1l+NkAX48mD+nU5s/xbJA==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.385.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.385.0.tgz", - "integrity": "sha512-rgqTANyiYNgdh2Hcgk8xFt/rmLCMGMnIlnZe+/9z2oggfkOKB9oAt49cyCLD9MZeXdTJuNPKn+IVMeSAOn6WYQ==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.397.2.tgz", + "integrity": "sha512-4DhxElRiz9FuwTcMMBhIfk+9cD/oA2+Nf0EBh5BZ1VK4EOD1BVutkTyzWyWP6J+Zqv4m91VyfdVK0Td0qXl+cw==" }, "node_modules/windmill-parser-wasm-rust": { - "version": "1.387.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-rust/-/windmill-parser-wasm-rust-1.387.0.tgz", - "integrity": "sha512-PX0Y/sdaaCkc9rrXB6m+AntGq/RX21fYhdpC7MFCMKb9VakkDAGnFo435xhAi74yLarGd03odUxGymokcBfU6g==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-rust/-/windmill-parser-wasm-rust-1.397.2.tgz", + "integrity": "sha512-7tjekDV2yuqqSjpPF/BfRzaXUwT5eckMWCuq4o18UzChOOeeA4U5goY4INBOGFzYR89m9AeMhwT1IM71nOGrIA==" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.385.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.385.0.tgz", - "integrity": "sha512-DKQZQzJmSqi7jCbROl9B8ZE/T6sG951zxs1/StWAJmnQazA0FJrdjTj0niiFQNkLaQ2aAPZUkSULZJG+7x1Jow==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.397.2.tgz", + "integrity": "sha512-nspEC1zLfrvvh6ta8YSjKo99lDfFvsXh7rV2mmpTAmc2ZzzyuVDvW5Cm0Wmo8Ww3seO+oDi/v3jy+8QX8pIzwA==" }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.396.1", diff --git a/frontend/package.json b/frontend/package.json index 910d85ecea..c2c85312ab 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.397.1", + "version": "1.398.1", "scripts": { "dev": "vite dev", "build": "vite build", @@ -139,12 +139,12 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.0.8", "vscode-ws-jsonrpc": "~3.3.2", - "windmill-parser-wasm-go": "^1.382.2", - "windmill-parser-wasm-php": "^1.382.2", - "windmill-parser-wasm-py": "^1.382.2", - "windmill-parser-wasm-regex": "^1.382.2", - "windmill-parser-wasm-rust": "^1.387.0", - "windmill-parser-wasm-ts": "^1.382.2", + "windmill-parser-wasm-go": "^1.397.2", + "windmill-parser-wasm-php": "^1.397.2", + "windmill-parser-wasm-py": "^1.397.2", + "windmill-parser-wasm-regex": "^1.397.2", + "windmill-parser-wasm-rust": "^1.397.2", + "windmill-parser-wasm-ts": "^1.397.2", "windmill-parser-wasm-yaml": "^1.396.1", "windmill-sql-datatype-parser-wasm": "^1.318.0", "y-monaco": "^0.1.4", @@ -459,4 +459,4 @@ "optionalDependencies": { "fsevents": "^2.3.3" } -} +} \ No newline at end of file diff --git a/frontend/src/lib/common.ts b/frontend/src/lib/common.ts index ba67c7d651..b96af76d03 100644 --- a/frontend/src/lib/common.ts +++ b/frontend/src/lib/common.ts @@ -143,7 +143,7 @@ export type IntRange = | Exclude, Enumerate> | T -export function pathToMeta(path: string): Meta { +export function pathToMeta(path: string, hideUser: boolean): Meta { const splitted = path.split('/') let ownerKind: OwnerKind if (splitted[0] == 'g') { @@ -155,7 +155,7 @@ export function pathToMeta(path: string): Meta { } else { console.error('Not recognized owner:' + splitted[0]) return { - ownerKind: 'user', + ownerKind: hideUser ? 'folder' : 'user', owner: '', name: '' } diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index b6942a7bfd..fbf7fcc0bb 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -485,6 +485,7 @@ return [ 'settings-metadata', 'constants', + 'preprocessor', ...dfsApply($flowStore.value.modules, (module) => module.id) ] } @@ -1322,6 +1323,7 @@ on:reload={() => { renderCount += 1 }} + {newFlow} /> {:else} Loading... diff --git a/frontend/src/lib/components/FlowGraphViewer.svelte b/frontend/src/lib/components/FlowGraphViewer.svelte index 9943deb9d0..b3bc035fa4 100644 --- a/frontend/src/lib/components/FlowGraphViewer.svelte +++ b/frontend/src/lib/components/FlowGraphViewer.svelte @@ -38,6 +38,7 @@ minHeight={400} modules={flow?.value?.modules} failureModule={flow?.value?.failure_module} + preprocessorModule={flow?.value?.preprocessor_module} on:select={(e) => { const mod = dfs(flow?.value?.modules ?? [], (m) => m).find((m) => m?.id === e?.detail) stepDetail = mod ?? e.detail diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 8dd837b2f3..0232548183 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -148,6 +148,9 @@ status?.modules?.concat( status.failure_module.type != 'WaitingForPriorSteps' ? status.failure_module : [] ) ?? [] + if (status.preprocessor_module) { + innerModules.unshift(status.preprocessor_module) + } updateInnerModules() let count = status.retry?.fail_count @@ -789,6 +792,7 @@ {/each} {:else if innerModules.length > 0 && (job.raw_flow?.modules.length ?? 0) > 0} + {@const hasPreprocessor = innerModules[0]?.id == 'preprocessor' ? 1 : 0}

    Step-by-step @@ -798,10 +802,12 @@ {#if render}

    - {#if job?.raw_flow?.modules && i < job?.raw_flow?.modules.length} + {#if mod.id === 'preprocessor'} +

    Preprocessor module

    + {:else if job?.raw_flow?.modules && i < job?.raw_flow?.modules.length + hasPreprocessor} Step - {i + 1} + {i + 1 - hasPreprocessor} out of {job?.raw_flow?.modules.length} @@ -973,6 +979,8 @@ globalRefreshes[detail.moduleId]?.({ job: detail.id, index: detail.index }) }} modules={job.raw_flow?.modules ?? []} + failureModule={job.raw_flow?.failure_module} + preprocessorModule={job.raw_flow?.preprocessor_module} />
    import { settings, settingsKeys, type SettingStorage } from './instanceSettings' import { Button, Skeleton, Tab, TabContent, Tabs } from '$lib/components/common' - import { ConfigService, SettingService, SettingsService } from '$lib/gen' + import { SettingService, SettingsService } from '$lib/gen' import Toggle from '$lib/components/Toggle.svelte' import SecondsInput from '$lib/components/common/seconds/SecondsInput.svelte' import Tooltip from '$lib/components/Tooltip.svelte' @@ -54,7 +54,6 @@ attempted_at: string } | null = null - let serverConfig = {} let initialValues: Record = {} let loading = true @@ -71,17 +70,10 @@ async function loadSettings() { loading = true - try { - serverConfig = (await ConfigService.getConfig({ name: 'server' })) ?? {} - } catch (e) { - console.log("Sever config not found, assuming it's first setup") - } async function getValue(key: string, storage: SettingStorage) { if (storage == 'setting') { return SettingService.getGlobal({ key }) - } else if (storage == 'config') { - return serverConfig[key] } } initialOauths = (await SettingService.getGlobal({ key: 'oauths' })) ?? {} @@ -110,6 +102,9 @@ if (values['base_url'] == undefined) { values['base_url'] = 'http://localhost' } + if (values['smtp_connect'] == undefined) { + values['smtp_connect'] = {} + } loading = false latestKeyRenewalAttempt = await SettingService.getLatestKeyRenewalAttempt() @@ -118,18 +113,6 @@ export async function saveSettings() { if (values) { const allSettings = Object.values(settings).flatMap((x) => Object.entries(x)) - const newServerConfig = Object.fromEntries( - allSettings - .filter((x) => x[1].storage == 'config' && values?.[x[1].key] && values?.[x[1].key] != '') - .map((x) => [x[1].key, values?.[x[1].key]]) - ) - if (!deepEqual(newServerConfig, serverConfig)) { - await ConfigService.updateConfig({ - name: 'server', - requestBody: newServerConfig - }) - serverConfig = JSON.parse(JSON.stringify(newServerConfig)) - } let licenseKeySet = false await Promise.all( allSettings @@ -291,29 +274,45 @@ {#if category == 'SMTP'}
    Setting SMTP unlocks sending emails upon adding new users to the workspace or the - instance.
    Setting SMTP unlocks sending emails upon adding new users to the workspace or the instance or sending critical alerts. + Learn more
    + {:else if category == 'Registries'} +
    + Add private registries for Pip, Bun and npm. Learn more +
    {:else if category == 'Slack'}
    - Connecting your instance to a Slack workspace enables critical alerts to be sent to a - Slack channel. + Connecting your instance to a Slack workspace enables critical alerts to be sent to a Slack channel. + Learn more +
    + {:else if category == 'SCIM/SAML'} +
    + Setting up SAML and SCIM allows you to authenticate users using your identity provider. + Learn more +
    + {:else if category == 'Debug'} +
    + Enable debug mode to get more detailed logs.
    {:else if category == 'Telemetry'}
    Anonymous usage data is collected to help improve Windmill.
    The following information is collected:
      -
    • version
      • +
    • version of your instance
    • number and total duration of jobs
    • accounts usage
    • login type usage
    • workers usage
      • -
    • vcpus usage
      • +
    • vCPUs usage
    • memory usage
    {#if $enterpriseLicense} +
    + On Enterprise Edition, you must send data to check that usage is in line with the terms of the subscription. You can either enable telemetry or regularly send usage data by clicking the button below. +
    +
    +
    +
    + +
    + + {#if $enterpriseLicense && Array.isArray(values[setting.key])} + {#each values[setting.key] ?? [] as v, i} +
    + + {#if v && 'slack_channel' in v} + { + if (e.target?.['value']) { + values[setting.key][i] = { + slack_channel: e.target['value'] + } + } + }} + value={v?.slack_channel ?? ''} + /> + {:else} + { + if (e.target?.['value']) { + values[setting.key][i] = { + email: e.target['value'] + } + } + }} + value={v?.email ?? ''} + /> + {/if} + +
    + {/each} + {/if}
    - {#if $enterpriseLicense && Array.isArray(values[setting.key])} - {#each values[setting.key] ?? [] as v, i} -
    - - {#if v && 'slack_channel' in v} - { - if (e.target?.['value']) { - values[setting.key][i] = { - slack_channel: e.target['value'] - } - } - }} - value={v?.slack_channel ?? ''} - /> - {:else} - { - if (e.target?.['value']) { - values[setting.key][i] = { - email: e.target['value'] - } - } - }} - value={v?.email ?? ''} - /> - {/if} - -
    - {/each} - {/if} + } catch (error) { + sendUserToast( + 'Failed to send test message: ' + error.message, + true + ) + } + }} + > + Test Critical Channels + +
    +
    + {:else if setting.fieldType == 'smtp_connect'} +
    + {#if values[setting.key]} +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + + +
    +
    + +
    + {/if} +
    {:else if setting.fieldType == 'object_store_config'}
    @@ -863,21 +947,22 @@
    {#if category == 'SMTP'} + {@const smtp = values['smtp_settings']}
    {#if bucket_config} -
    +
    + {#if !$copilotInfo.exists_openai_resource_path} +
    +

    + Enable Windmill AI in the + { + close() + }} + > + workspace settings + + +

    +
    + {/if} + +
    + + AI Flow builder + +
    diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index dbdc85d37e..8e75c6622e 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -224,8 +224,8 @@ {/if}
    {label}
    -
    - {#if id} +
    + {#if id && id !== 'preprocessor' && !id.startsWith('failure')} {id} {#if deletable} @@ -253,6 +253,7 @@ hover:border-blue-700 hover:!visible {hover ? '' : '!hidden'}" + {#if id !== 'preprocessor'} + {/if} {#if (id && Object.values($flowInputsStore?.[id]?.flowStepWarnings || {}).length > 0) || Boolean(warningMessage)}
    diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte index f51c1f0256..bdc3552d40 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte @@ -31,6 +31,8 @@ import { tutorialInProgress } from '$lib/tutorialUtils' import FlowGraphV2 from '$lib/components/graph/FlowGraphV2.svelte' import { replaceId } from '../flowStore' + import { emptySchema } from '$lib/utils' + import { NEVER_TESTED_THIS_FAR } from '../models' export let modules: FlowModule[] | undefined export let sidebarSize: number | undefined = undefined @@ -38,7 +40,7 @@ export let disableTutorials = false export let disableAi = false export let disableSettings = false - + export let newFlow: boolean = false export let smallErrorHandler = false let flowTutorials: FlowTutorials | undefined = undefined @@ -256,6 +258,7 @@ { @@ -270,13 +274,15 @@ dependents = getDependentComponents(e.id, $flowStore) const cb = () => { push(history, $flowStore) - - selectNextId(e.id) - - removeAtId($flowStore.value.modules, e.id) - - if ($flowInputsStore) { - delete $flowInputsStore[e.id] + if (e.id === 'preprocessor') { + $selectedId = 'Input' + $flowStore.value.preprocessor_module = undefined + } else { + selectNextId(e.id) + removeAtId($flowStore.value.modules, e.id) + if ($flowInputsStore) { + delete $flowInputsStore[e.id] + } } $flowStore = $flowStore @@ -307,13 +313,26 @@ $selectedId = $moving.module.id $moving = undefined } else { - await insertNewModuleAtIndex( - detail.modules, - detail.index ?? 0, - detail.detail, - detail.script - ) - $selectedId = detail.modules[detail.index ?? 0].id + if (detail.detail === 'preprocessor') { + const preprocessorModule = { + schema: emptySchema(), + previewResult: NEVER_TESTED_THIS_FAR + } + $flowStore.value.preprocessor_module = { + id: 'preprocessor', + value: { type: 'identity' } + } + $flowStateStore['preprocessor'] = preprocessorModule + $selectedId = 'preprocessor' + } else { + await insertNewModuleAtIndex( + detail.modules, + detail.index ?? 0, + detail.detail, + detail.script + ) + $selectedId = detail.modules[detail.index ?? 0].id + } } if (['branchone', 'branchall'].includes(detail.detail)) { diff --git a/frontend/src/lib/components/flows/map/MapItem.svelte b/frontend/src/lib/components/flows/map/MapItem.svelte index c7c78e6a84..a171cd3344 100644 --- a/frontend/src/lib/components/flows/map/MapItem.svelte +++ b/frontend/src/lib/components/flows/map/MapItem.svelte @@ -11,9 +11,11 @@ import { msToSec } from '$lib/utils' import BarsStaggered from '$lib/components/icons/BarsStaggered.svelte' import FlowJobsMenu from './FlowJobsMenu.svelte' + import FlowCopilotButton from './FlowCopilotButton.svelte' export let mod: FlowModule export let insertable: boolean + export let disableAi: boolean export let annotation: string | undefined = undefined export let bgColor: string = '' export let modules: FlowModule[] @@ -60,6 +62,9 @@ {#if mod}
    + {#if insertable && !disableAi && mod.id === 'preprocessor'} + + {/if} {#if moving == mod.id}
    - {#if !$copilotInfo.exists_openai_resource_path} -
    -

    - Enable Windmill AI in the - { - close() - }} - > - workspace settings - - -

    -
    - {/if} - -
    - - AI Flow builder - +
    {/if} { data.eventHandlers.delete(e.detail, '') diff --git a/frontend/src/lib/components/graph/renderers/nodes/TriggersNode.svelte b/frontend/src/lib/components/graph/renderers/nodes/TriggersNode.svelte index a505b6b99d..58d71b00f2 100644 --- a/frontend/src/lib/components/graph/renderers/nodes/TriggersNode.svelte +++ b/frontend/src/lib/components/graph/renderers/nodes/TriggersNode.svelte @@ -7,6 +7,7 @@ openSchedules: () => void triggerDetail: (e: { detail: 'webhook' | 'mail' | 'schedule' }) => void isEditor: boolean + newFlow: boolean } @@ -18,5 +19,6 @@ data.triggerDetail(e.detail) }} isEditor={data.isEditor} + newFlow={data.newFlow} /> diff --git a/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte b/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte index 5ac4a55f47..60c81c447e 100644 --- a/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte +++ b/frontend/src/lib/components/graph/renderers/triggers/TriggersWrapper.svelte @@ -2,8 +2,8 @@ import { Calendar, Mail, Webhook } from 'lucide-svelte' import TriggerButton from './TriggerButton.svelte' import { NODE } from '../../util' - import { ScheduleService, type Schedule } from '$lib/gen' - import { workspaceStore } from '$lib/stores' + import { HttpTriggerService, ScheduleService, type HttpTrigger, type Schedule } from '$lib/gen' + import { userStore, workspaceStore } from '$lib/stores' import Popover from '$lib/components/Popover.svelte' import Drawer from '$lib/components/common/drawer/Drawer.svelte' import { DrawerContent, Tabs } from '$lib/components/common' @@ -13,11 +13,16 @@ import TriggerCount from './TriggerCount.svelte' import Tooltip from '$lib/components/Tooltip.svelte' import { createEventDispatcher } from 'svelte' + import { Route } from 'lucide-svelte' + import { canWrite } from '$lib/utils' + import RoutesPanel from '$lib/components/triggers/RoutesPanel.svelte' let schedules: Schedule[] | undefined = undefined + let triggers: (HttpTrigger & { canWrite: boolean })[] | undefined = undefined export let path: string export let isEditor: boolean + export let newFlow: boolean const dispatch = createEventDispatcher() @@ -34,92 +39,138 @@ } } + async function loadTriggers() { + try { + triggers = ( + await HttpTriggerService.listHttpTriggers({ + workspace: $workspaceStore ?? '', + path, + isFlow: true + }) + ).map((x) => { + return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } + }) + } catch (e) { + console.error('impossible to load http routes') + } + } + loadSchedules() + loadTriggers() let drawer: Drawer | undefined = undefined - let selectedTab: 'webhooks' | 'mail' | 'schedules' = 'webhooks' + let selectedTab: 'webhooks' | 'mail' | 'routes' = 'webhooks' -{#if isEditor} - - - - Webhooks - Mail - - {#if selectedTab === 'webhooks'} - - {/if} +{#if !newFlow} + {#if isEditor} + + + + Webhooks + Mail + Routes - {#if selectedTab === 'mail'} - - {/if} - - - - -{/if} + + {#if selectedTab === 'webhooks'} + + {/if} -
    -
    -
    -
    -
    - Triggers + {#if selectedTab === 'mail'} + + {/if} - - The flow can be triggered by webhooks, emails, or schedules. Click on the icons to see - the triggers. - + {#if selectedTab === 'routes'} + + {/if} + + + + +
    +
    +
    +
    +
    + Triggers + + + The flow can be triggered by webhooks, emails, or schedules. Click on the icons to + see the triggers. + +
    +
    + + See default webhooks triggers + { + if (isEditor) { + selectedTab = 'webhooks' + drawer?.openDrawer() + } else { + dispatch('triggerDetail', 'webhooks') + } + }} + > + + + + + See default mail trigger + { + if (isEditor) { + selectedTab = 'mail' + drawer?.openDrawer() + } else { + dispatch('triggerDetail', 'mail') + } + }} + > + + + + + See all routes + { + if (isEditor) { + selectedTab = 'routes' + drawer?.openDrawer() + } else { + dispatch('triggerDetail', 'routes') + } + }} + > + + + + + + See all schedules + { + if (isEditor) { + dispatch('openSchedules') + } else { + dispatch('triggerDetail', 'schedule') + } + }} + > + + + +
    - - See default webhooks triggers - { - if (isEditor) { - selectedTab = 'webhooks' - drawer?.openDrawer() - } else { - dispatch('triggerDetail', 'webhooks') - } - }} - > - - - - - See default mail trigger - { - if (isEditor) { - selectedTab = 'mail' - drawer?.openDrawer() - } else { - dispatch('triggerDetail', 'mail') - } - }} - > - - - - - See all schedules - { - if (isEditor) { - dispatch('openSchedules') - } else { - dispatch('triggerDetail', 'schedule') - } - }} - > - - - -
    -
    -
    + {/if} +{/if} diff --git a/frontend/src/lib/components/instanceSettings.ts b/frontend/src/lib/components/instanceSettings.ts index 21ee3cb098..4bb35371dc 100644 --- a/frontend/src/lib/components/instanceSettings.ts +++ b/frontend/src/lib/components/instanceSettings.ts @@ -19,19 +19,20 @@ export interface Setting { | 'object_store_config' | 'critical_error_channels' | 'slack_connect' + | 'smtp_connect' storage: SettingStorage isValid?: (value: any) => boolean error?: string defaultValue?: () => any } -export type SettingStorage = 'setting' | 'config' +export type SettingStorage = 'setting' export const settings: Record = { Core: [ { label: 'Base url', - description: 'Public base url of the instance', + description: 'Public base url of the instance. Learn more', key: 'base_url', fieldType: 'text', placeholder: 'https://windmill.com', @@ -47,7 +48,8 @@ export const settings: Record = { }, { label: 'Email domain', - description: 'Domain to display in webhooks for email triggers (should match the MX record)', + description: + 'Domain to display in webhooks for email triggers (should match the MX record)', key: 'email_domain', fieldType: 'text', storage: 'setting', @@ -65,22 +67,23 @@ export const settings: Record = { { label: 'Default timeout', key: 'job_default_timeout', - description: 'Default timeout for individual jobs', + description: 'Default timeout for individual jobs. Learn more', fieldType: 'seconds', storage: 'setting', cloudonly: false }, { label: 'Max timeout for sync endpoints', + description: 'Maximum amount of time (measured in seconds) that a sync endpoint is allowed to run before it is forcibly stopped or timed out.', key: 'timeout_wait_result', cloudonly: true, fieldType: 'seconds', placeholder: '60', - storage: 'config' + storage: 'setting' }, { label: 'License key', - description: 'License key required to use the EE (switch image for windmill-ee)', + description: 'License key required to use the EE (switch image for windmill-ee). Learn more', key: 'license_key', fieldType: 'license_key', placeholder: 'only needed to prepare upgrade to EE', @@ -88,7 +91,7 @@ export const settings: Record = { }, { label: 'Non-prod instance', - description: 'Whether we should consider the reported usage of this instance as non-prod', + description: 'Whether we should consider the reported usage of this instance as non-prod. Learn more', key: 'dev_instance', fieldType: 'boolean', storage: 'setting' @@ -96,7 +99,7 @@ export const settings: Record = { { label: 'Retention period in secs', key: 'retention_period_secs', - description: 'How long to keep the jobs data in the database (max 30 days on CE)', + description: 'How long to keep the jobs data in the database (max 30 days on CE). Learn more', fieldType: 'seconds', placeholder: '30', storage: 'setting', @@ -105,15 +108,15 @@ export const settings: Record = { }, { label: 'Expose metrics', - description: 'Expose Prometheus metrics for workers and servers on port 8001 at /metrics', + description: 'Expose Prometheus metrics for workers and servers on port 8001 at /metrics. Learn more', key: 'expose_metrics', fieldType: 'boolean', storage: 'setting', ee_only: '' }, { - label: 'S3/Azure for Python/Go cache & large logs', - description: 'Bucket to store large logs and global cache for Python and Go.', + label: 'Instance object storage', + description: ' S3/Azure bucket to store large logs and global cache for Python and Go. Learn more', key: 'object_store_cache_config', fieldType: 'object_store_config', storage: 'setting', @@ -121,8 +124,7 @@ export const settings: Record = { }, { label: 'Critical alert channels', - description: - 'Channels to send critical alerts to. SMTP must be configured for the email channel. A Slack workspace must be connected to the instance for the Slack channel.', + description: 'Channels to send critical alerts to. SMTP must be configured for the email channel. A Slack workspace must be connected to the instance for the Slack channel. Learn more', key: 'critical_error_channels', fieldType: 'critical_error_channels', storage: 'setting', @@ -130,8 +132,7 @@ export const settings: Record = { }, { label: 'Azure OpenAI base path', - description: - 'All Windmill AI features will run on the specified deployed model. Format: https://{your-resource-name}.openai.azure.com/openai/deployments/{deployment-id}', + description: 'All Windmill AI features will run on the specified deployed model. Format: https://{your-resource-name}.openai.azure.com/openai/deployments/{deployment-id}. Learn more', key: 'openai_azure_base_path', fieldType: 'text', storage: 'setting', @@ -139,7 +140,7 @@ export const settings: Record = { }, { label: 'Private Hub base url', - description: 'Base url of your private Hub instance, without trailing slash', + description: 'Base url of your private Hub instance, without trailing slash. Learn more', placeholder: 'https://hub.company.com', key: 'hub_base_url', fieldType: 'text', @@ -151,7 +152,7 @@ export const settings: Record = { Registries: [ { label: 'Pip Index Url', - description: 'Add private PIP registry', + description: 'Add private Pip registry', key: 'pip_index_url', fieldType: 'text', placeholder: 'https://username:password@pypi.company.com/simple', @@ -160,7 +161,7 @@ export const settings: Record = { }, { label: 'Pip Extra Index Url', - description: 'Add private extra PIP registry', + description: 'Add private extra Pip registry', key: 'pip_extra_index_url', fieldType: 'text', placeholder: 'https://username:password@pypi.company.com/simple', @@ -169,7 +170,7 @@ export const settings: Record = { }, { label: 'Npm Config Registry', - description: 'Add private NPM registry', + description: 'Add private npm registry', key: 'npm_config_registry', fieldType: 'text', placeholder: 'https://registry.npmjs.org/:_authToken=npm_FOOBAR', @@ -188,45 +189,12 @@ export const settings: Record = { } ], SMTP: [ - { - label: 'Host', - key: 'smtp_host', - fieldType: 'text', - placeholder: 'smtp.gmail.com', - storage: 'config' - }, - { - label: 'Port', - key: 'smtp_port', - fieldType: 'number', - placeholder: '587', - storage: 'config' - }, - { - label: 'Username', - key: 'smtp_username', - fieldType: 'text', - placeholder: 'ruben@windmill.dev', - storage: 'config' - }, - { - label: 'Password', - key: 'smtp_password', - fieldType: 'password', - storage: 'config' - }, - { - label: 'From Address', - key: 'smtp_from', - placeholder: 'noreply@windmill.dev', - fieldType: 'email', - storage: 'config' - }, - { - label: 'Implicit TLS', - key: 'smtp_tls_implicit', - fieldType: 'boolean', - storage: 'config' + { + label: 'SMTP', + key: 'smtp_settings', + fieldType: 'smtp_connect', + storage: 'setting', + ee_only: '', } ], Slack: [ @@ -263,14 +231,14 @@ export const settings: Record = { label: 'Keep job directories', key: 'keep_job_dir', fieldType: 'boolean', - tooltip: 'Keep Job directories after execution at /tmp/windmill//', + description: 'Keep Job directories after execution at /tmp/windmill//', storage: 'setting' }, { label: 'Expose debug metrics', key: 'expose_debug_metrics', fieldType: 'boolean', - tooltip: 'Expose additional metrics (require metrics to be enabled)', + description: 'Expose additional metrics (require metrics to be enabled)', storage: 'setting' } ], diff --git a/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte b/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte new file mode 100644 index 0000000000..27ab382b51 --- /dev/null +++ b/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte @@ -0,0 +1,77 @@ + + +{#if hasPreprocessedArgs} +
    + preprocessed args +
    +{/if} + + diff --git a/frontend/src/lib/components/sidebar/SidebarContent.svelte b/frontend/src/lib/components/sidebar/SidebarContent.svelte index df662f4966..c50b9a8700 100644 --- a/frontend/src/lib/components/sidebar/SidebarContent.svelte +++ b/frontend/src/lib/components/sidebar/SidebarContent.svelte @@ -1,6 +1,6 @@ + +{#if open} + +{/if} diff --git a/frontend/src/lib/components/triggers/RouteEditorInner.svelte b/frontend/src/lib/components/triggers/RouteEditorInner.svelte new file mode 100644 index 0000000000..2fa7998295 --- /dev/null +++ b/frontend/src/lib/components/triggers/RouteEditorInner.svelte @@ -0,0 +1,340 @@ + + + + + + {#if !drawerLoading && can_write} + + {/if} + + {#if drawerLoading} + + {:else} +
    +
    + +
    + +
    + {#if !($userStore?.is_admin || $userStore?.is_super_admin)} + + Route endpoints can only be edited by workspace admins + +
    + {/if} +
    + + + + + + + + + +
    +
    + + Full endpoint + + { + currentTarget.select() + }} + /> +
    +
    {routeError}
    +
    +
    +
    + +
    +

    + Pick a script or flow to be triggered
    + To handle headers, query or path parameters, add a preprocessor to your runnable. +

    +
    + + + {#if script_path === undefined} + + {/if} +
    +
    +
    +
    +
    +
    Request type
    + + + + +
    +
    +
    Authentication
    + + + + +
    +
    +
    +
    + {/if} +     +     diff --git a/frontend/src/lib/components/triggers/RoutesPanel.svelte b/frontend/src/lib/components/triggers/RoutesPanel.svelte new file mode 100644 index 0000000000..cf37612d20 --- /dev/null +++ b/frontend/src/lib/components/triggers/RoutesPanel.svelte @@ -0,0 +1,81 @@ + + + { + loadTriggers() + }} + bind:this={routeEditor} +/> + +
    + {#if $userStore?.is_admin || $userStore?.is_super_admin} + + {/if} +
    + +{#if triggers} + {#if triggers.length == 0} +
    No http routes
    + {:else} +
    + {#each triggers as trigger (trigger.path)} +
    +
    {trigger.path}
    +
    + {trigger.http_method.toUpperCase()} /{trigger.route_path} +
    + +
    + {/each} +
    + {/if} +{:else} + +{/if} diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 86df88616a..0bb53957f5 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -32,7 +32,8 @@ import wasmUrlYaml from 'windmill-parser-wasm-yaml/windmill_parser_wasm_bg.wasm? import { workspaceStore } from './stores.js' import { argSigToJsonSchemaType } from './inferArgSig.js' -const loadSchemaLastRun = writable<[string | undefined, MainArgSignature | undefined]>(undefined) +const loadSchemaLastRun = + writable<[string | undefined, MainArgSignature | undefined, string | undefined]>(undefined) let initializeTsPromise: Promise | undefined = undefined export async function initWasmTs() { @@ -63,11 +64,15 @@ async function initWasmYaml() { export async function inferArgs( language: SupportedLanguage | 'bunnative' | undefined, code: string, - schema: Schema -): Promise { + schema: Schema, + mainOverride?: string +): Promise<{ + no_main_func: boolean | null + has_preprocessor: boolean | null +} | null> { const lastRun = get(loadSchemaLastRun) let inferedSchema: MainArgSignature - if (lastRun && code == lastRun[0] && lastRun[1]) { + if (lastRun && code == lastRun[0] && lastRun[1] && lastRun[2] == mainOverride) { inferedSchema = lastRun[1] } else { if (code == '') { @@ -81,16 +86,16 @@ export async function inferArgs( } if (language == 'python3') { await initWasmPython() - inferedSchema = JSON.parse(parse_python(code)) + inferedSchema = JSON.parse(parse_python(code, mainOverride)) } else if (language == 'deno') { await initWasmTs() - inferedSchema = JSON.parse(parse_deno(code)) + inferedSchema = JSON.parse(parse_deno(code, mainOverride)) } else if (language == 'nativets') { await initWasmTs() - inferedSchema = JSON.parse(parse_deno(code)) + inferedSchema = JSON.parse(parse_deno(code, mainOverride)) } else if (language == 'bun' || language == 'bunnative') { await initWasmTs() - inferedSchema = JSON.parse(parse_deno(code)) + inferedSchema = JSON.parse(parse_deno(code, mainOverride)) } else if (language == 'postgresql') { inferedSchema = JSON.parse(parse_sql(code)) if (inlineDBResource === undefined) { @@ -162,7 +167,7 @@ export async function inferArgs( if (inferedSchema.type == 'Invalid') { throw new Error(inferedSchema.error) } - loadSchemaLastRun.set([code, inferedSchema]) + loadSchemaLastRun.set([code, inferedSchema, mainOverride]) } schema.required = [] @@ -186,7 +191,10 @@ export async function inferArgs( } await tick() - return inferedSchema.no_main_func + return { + no_main_func: inferedSchema.no_main_func, + has_preprocessor: inferedSchema.has_preprocessor + } } export async function loadSchemaFromPath(path: string, hash?: string): Promise { diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 2cc38b6a54..458499d3d2 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -287,8 +287,6 @@ fn main(who_to_greet: String, numbers: Vec) -> anyhow::Result { } ` - - export const FETCH_INIT_CODE = `export async function main( url: string | undefined, method: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'OPTIONS' = 'GET', @@ -431,6 +429,48 @@ export async function main(approver?: string) { // add a form in Advanced - Suspend // all on approval steps: https://www.windmill.dev/docs/flows/flow_approval` +export const BUN_PREPROCESSOR_MODULE_CODE = ` +export async function preprocessor( + wm_trigger: { + kind: 'http' | 'email' | 'webhook', + http?: { + route: string // The route path, e.g. "/users/:id" + path: string // The actual path called, e.g. "/users/123" + method: string + params: Record + query: Record + headers: Record + } + }, + /* your other args */ +) { + return { + // return the args to be passed to the flow + } +} +` + +export const DENO_PREPROCESSOR_MODULE_CODE = ` +export async function preprocessor( + wm_trigger: { + kind: 'http' | 'email' | 'wehbook', + http?: { + route: string // The route path, e.g. "/users/:id" + path: string // The actual path called, e.g. "/users/123" + method: string + params: Record + query: Record + headers: Record + } + }, + /* your other args */ +) { + return { + // return the args to be passed to the flow + } +} +` + export const PYTHON_INIT_CODE_APPROVAL = `import wmill def main(): @@ -458,6 +498,29 @@ def main(): # add a form in Advanced - Suspend # all on approval steps: https://www.windmill.dev/docs/flows/flow_approval` +export const PYTHON_PREPROCESSOR_MODULE_CODE = `from typing import TypedDict, Literal + +class Http(TypedDict): + route: str # The route path, e.g. "/users/:id" + path: str # The actual path called, e.g. "/users/123" + method: str + params: dict[str, str] + query: dict[str, str] + headers: dict[str, str] + +class WmTrigger(TypedDict): + kind: Literal["http", "email", "webhook"] + http: Http | None + +def preprocessor( + wm_trigger: WmTrigger, + # your other args +): + return { + # return the args to be passed to the flow + } +` + export const DOCKER_INIT_CODE = `# shellcheck shell=bash # Bash script that calls docker as a client to the host daemon # See documentation: https://www.windmill.dev/docs/advanced/docker @@ -540,16 +603,21 @@ const ALL_INITIAL_CODE = [ DENO_INIT_CODE_TRIGGER, DENO_INIT_CODE_CLEAR, PYTHON_INIT_CODE_CLEAR, + PYTHON_FAILURE_MODULE_CODE, + PYTHON_PREPROCESSOR_MODULE_CODE, DENO_INIT_CODE_APPROVAL, DENO_FAILURE_MODULE_CODE, + DENO_PREPROCESSOR_MODULE_CODE, BUN_INIT_CODE, BUN_INIT_CODE_CLEAR, BUN_INIT_CODE_APPROVAL, + BUN_FAILURE_MODULE_CODE, + BUN_PREPROCESSOR_MODULE_CODE, BASH_INIT_CODE, POWERSHELL_INIT_CODE, PHP_INIT_CODE, RUST_INIT_CODE, - ANSIBLE_PLAYBOOK_INIT_CODE, + ANSIBLE_PLAYBOOK_INIT_CODE ] export function isInitialCode(content: string): boolean { @@ -573,6 +641,7 @@ export function initialCode( | 'docker' | 'powershell' | 'bunnative' + | 'preprocessor' | undefined ): string { if (!kind) { @@ -590,6 +659,8 @@ export function initialCode( return MYSQL_INIT_CODE } else if (subkind === 'fetch') { return FETCH_INIT_CODE + } else if (subkind === 'preprocessor') { + return DENO_PREPROCESSOR_MODULE_CODE } else { return DENO_INIT_CODE } @@ -609,6 +680,8 @@ export function initialCode( return PYTHON_FAILURE_MODULE_CODE } else if (subkind === 'flow') { return PYTHON_INIT_CODE_CLEAR + } else if (subkind === 'preprocessor') { + return PYTHON_PREPROCESSOR_MODULE_CODE } else { return PYTHON_INIT_CODE } @@ -647,6 +720,8 @@ export function initialCode( return BUN_INIT_CODE_APPROVAL } else if (kind === 'failure') { return BUN_FAILURE_MODULE_CODE + } else if (subkind === 'preprocessor') { + return BUN_PREPROCESSOR_MODULE_CODE } if (subkind === 'flow') { return BUN_INIT_CODE_CLEAR diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index aa65ceb8bc..18cf32b074 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -89,6 +89,7 @@ export const vimMode = writable(getLocalSetting(VIM_MODE_SETTING_NAME) export const codeCompletionSessionEnabled = writable( getLocalSetting(CODE_COMPLETION_SETTING_NAME) != 'false' ) +export const usedTriggerKinds = writable([]) type SQLBaseSchema = { [schemaKey: string]: { diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 9b9f6ec35c..3eae42916f 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -8,6 +8,7 @@ RawAppService, ScriptService, SettingService, + HttpTriggerService, UserService, WorkspaceService } from '$lib/gen' @@ -25,7 +26,8 @@ workspaceStore, type UserExt, defaultScripts, - hubBaseUrlStore + hubBaseUrlStore, + usedTriggerKinds } from '$lib/stores' import CenteredModal from '$lib/components/CenteredModal.svelte' import { afterNavigate, beforeNavigate } from '$app/navigation' @@ -124,6 +126,7 @@ loadUsage() syncTutorialsTodos() loadHubBaseUrl() + loadUsedTriggerKinds() } async function loadUsage() { @@ -183,6 +186,11 @@ ] } + async function loadUsedTriggerKinds() { + const httpUsed = await HttpTriggerService.used({ workspace: $workspaceStore ?? '' }) + $usedTriggerKinds = httpUsed ? ['http'] : [] + } + function pathInAppMode(pathname: string | undefined): boolean { if (!pathname) return false return ( diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 7e73c98b73..a2e97450e9 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -55,6 +55,7 @@ import FlowHistory from '$lib/components/flows/FlowHistory.svelte' import EmailTriggerPanel from '$lib/components/details/EmailTriggerPanel.svelte' import Star from '$lib/components/Star.svelte' + import RoutesPanel from '$lib/components/triggers/RoutesPanel.svelte' let flow: Flow | undefined let can_write = false @@ -130,7 +131,8 @@ invisibleToOwner, requestBody: args, scheduledFor, - tag: overrideTag + tag: overrideTag, + skipPreprocessor: true }) await goto('/run/' + run + '?workspace=' + $workspaceStore) } catch (e) { @@ -496,6 +498,9 @@ {args} /> + + + + import { HttpTriggerService, type HttpTrigger } from '$lib/gen' + import { canWrite, displayDate, getLocalSetting, storeLocalSetting } from '$lib/utils' + import { base } from '$app/paths' + import CenteredPage from '$lib/components/CenteredPage.svelte' + import { Button, Skeleton } from '$lib/components/common' + import Dropdown from '$lib/components/DropdownV2.svelte' + import PageHeader from '$lib/components/PageHeader.svelte' + import RouteEditor from '$lib/components/triggers/RouteEditor.svelte' + import SharedBadge from '$lib/components/SharedBadge.svelte' + import ShareModal from '$lib/components/ShareModal.svelte' + import Toggle from '$lib/components/Toggle.svelte' + import { userStore, workspaceStore } from '$lib/stores' + import { Route, Code, Eye, Pen, Plus, Share, Trash } from 'lucide-svelte' + import { goto } from '$lib/navigation' + import SearchItems from '$lib/components/SearchItems.svelte' + import NoItemFound from '$lib/components/home/NoItemFound.svelte' + import RowIcon from '$lib/components/common/table/RowIcon.svelte' + import ListFilters from '$lib/components/home/ListFilters.svelte' + import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' + import { setQuery } from '$lib/navigation' + import { onMount } from 'svelte' + + type TriggerW = HttpTrigger & { canWrite: boolean } + + let triggers: TriggerW[] = [] + let shareModal: ShareModal + let loading = true + + async function loadTriggers(): Promise { + triggers = (await HttpTriggerService.listHttpTriggers({ workspace: $workspaceStore! })).map( + (x) => { + return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } + } + ) + loading = false + } + + $: { + if ($workspaceStore && $userStore) { + loadTriggers() + } + } + let routeEditor: RouteEditor + + let filteredItems: (TriggerW & { marked?: any })[] | undefined = [] + let items: typeof filteredItems | undefined = [] + let preFilteredItems: typeof filteredItems | undefined = [] + let filter = '' + let ownerFilter: string | undefined = undefined + let nbDisplayed = 15 + + const TRIGGER_PATH_KIND_FILTER_SETTING = 'filter_path_of' + const FILTER_USER_FOLDER_SETTING_NAME = 'user_and_folders_only' + let selectedFilterKind = + (getLocalSetting(TRIGGER_PATH_KIND_FILTER_SETTING) as 'route' | 'script_flow') ?? 'route' + let filterUserFolders = getLocalSetting(FILTER_USER_FOLDER_SETTING_NAME) == 'true' + + $: storeLocalSetting(TRIGGER_PATH_KIND_FILTER_SETTING, selectedFilterKind) + $: storeLocalSetting(FILTER_USER_FOLDER_SETTING_NAME, filterUserFolders ? 'true' : undefined) + + function filterItemsPathsBaseOnUserFilters( + item: TriggerW, + selectedFilterKind: 'route' | 'script_flow', + filterUserFolders: boolean + ) { + if ($workspaceStore == 'admins') return true + if (filterUserFolders) { + if (selectedFilterKind === 'route') { + return ( + !item.path.startsWith('u/') || item.path.startsWith('u/' + $userStore?.username + '/') + ) + } else { + return ( + !item.script_path.startsWith('u/') || + item.script_path.startsWith('u/' + $userStore?.username + '/') + ) + } + } else { + return true + } + } + + $: preFilteredItems = + ownerFilter != undefined + ? selectedFilterKind === 'route' + ? triggers?.filter( + (x) => + x.path.startsWith(ownerFilter + '/' ?? '') && + filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) + ) + : triggers?.filter( + (x) => + x.script_path.startsWith(ownerFilter + '/' ?? '') && + filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) + ) + : triggers?.filter((x) => + filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) + ) + + $: if ($workspaceStore) { + ownerFilter = undefined + } + + $: owners = + selectedFilterKind === 'route' + ? Array.from( + new Set(filteredItems?.map((x) => x.path.split('/').slice(0, 2).join('/')) ?? []) + ).sort() + : Array.from( + new Set(filteredItems?.map((x) => x.script_path.split('/').slice(0, 2).join('/')) ?? []) + ).sort() + + $: items = filter !== '' ? filteredItems : preFilteredItems + + function updateQueryFilters(selectedFilterKind, filterUserFolders) { + setQuery( + new URL(window.location.href), + TRIGGER_PATH_KIND_FILTER_SETTING, + selectedFilterKind + ).then(() => { + setQuery( + new URL(window.location.href), + FILTER_USER_FOLDER_SETTING_NAME, + String(filterUserFolders) + ) + }) + } + + function loadQueryFilters() { + let url = new URL(window.location.href) + let queryFilterKind = url.searchParams.get(TRIGGER_PATH_KIND_FILTER_SETTING) + let queryFilterUserFolders = url.searchParams.get(FILTER_USER_FOLDER_SETTING_NAME) + if (queryFilterKind) { + selectedFilterKind = queryFilterKind as 'route' | 'script_flow' + } + if (queryFilterUserFolders) { + filterUserFolders = queryFilterUserFolders == 'true' + } + } + + onMount(() => { + loadQueryFilters() + }) + + $: updateQueryFilters(selectedFilterKind, filterUserFolders) + + + + + (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} +/> + + + + {#if $userStore?.is_admin || $userStore?.is_super_admin} + + {/if} + +
    +
    + +
    +
    Filter by path of
    + + + + +
    + + +
    + {#if $userStore?.is_super_admin && $userStore.username.includes('@')} + + {:else if $userStore?.is_admin || $userStore?.is_super_admin} + + {/if} +
    +
    + {#if loading} + {#each new Array(6) as _} + + {/each} + {:else if !triggers?.length} +
    No routes
    + {:else if items?.length} +
    + {#each items.slice(0, nbDisplayed) as { path, edited_by, edited_at, script_path, route_path, is_flow, extra_perms, canWrite, marked, http_method } (path)} + {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} + +
    +
    + + + routeEditor?.openEdit(path, is_flow)} + class="min-w-0 grow hover:underline decoration-gray-400" + > +
    + {#if marked} + + {@html marked} + + {:else} + {http_method.toUpperCase()} /{route_path} + {/if} +
    +
    + {path} +
    +
    + runnable: {script_path} +
    +     + + + +
    + + { + goto(href) + } + }, + { + displayName: 'Delete', + type: 'delete', + icon: Trash, + disabled: !canWrite || !($userStore?.is_admin || $userStore?.is_super_admin), + action: async () => { + await HttpTriggerService.deleteHttpTrigger({ + workspace: $workspaceStore ?? '', + path + }) + loadTriggers() + } + }, + { + displayName: canWrite ? 'Edit' : 'View', + icon: canWrite ? Pen : Eye, + action: () => { + routeEditor?.openEdit(path, is_flow) + } + }, + { + displayName: 'Audit logs', + icon: Eye, + href: `${base}/audit_logs?resource=${path}` + }, + { + displayName: canWrite ? 'Share' : 'See Permissions', + icon: Share, + action: () => { + shareModal.openDrawer(path, 'http_trigger') + } + } + ]} + /> +
    +
    +
    +
    edited by {edited_by}
    the {displayDate(edited_at)}
    +
    + {/each} +
    + {:else} + + {/if} +
    + {#if items && items?.length > 15 && nbDisplayed < items.length} + {nbDisplayed} items out of {items.length} + + {/if} +     + + { + loadTriggers() + }} +/> diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 4ae0562272..8ab81a1e38 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -91,6 +91,7 @@ import ScheduleEditor from '$lib/components/ScheduleEditor.svelte' import Popover from '$lib/components/Popover.svelte' import HighlightTheme from '$lib/components/HighlightTheme.svelte' + import PreprocessedArgsDisplay from '$lib/components/runs/PreprocessedArgsDisplay.svelte' let job: Job | undefined let jobUpdateLastFetch: Date | undefined @@ -608,7 +609,11 @@
    {/if} + {#if job && job.flow_status && job.job_kind === 'script'} + + {/if} {#if persistentScriptDefinition} + - + (saved = true)} />

    You can change these settings later in the instance settings but finishing setup will leave this diff --git a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte index 4913f6b1e8..6f7b9dcae9 100644 --- a/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/workspace_settings/+page.svelte @@ -568,6 +568,7 @@ let jobId = await JobService.runScriptByPath({ workspace: $workspaceStore!, path: hubPaths.gitSyncTest, + skipPreprocessor: true, requestBody: { repo_url_resource_path: gitSyncRepository.git_repo_resource_path.replace('$res:', '') } @@ -685,7 +686,7 @@

    Windmill AI
    -
    S3 Storage
    +
    Object Storage (S3)
    Default App
    @@ -1052,11 +1053,11 @@ {:else if tab == 'windmill_lfs'}
    -
    S3 Storage
    +
    Workspace object storage (S3/Azure Blob)
    - Connect your Windmill workspace to your S3 bucket or your Azure Blob storage. + Connect your Windmill workspace to your S3 bucket or your Azure Blob storage to enable users to read and write from S3 without having to have access to the credentials. Learn more. @@ -1073,7 +1074,7 @@ This setting is only for storage of large files allowing to upload files directly to object storage using S3Object and use the wmill sdk to read and write large files backed - by an object storage. The automatics large logs storage is set by the superadmins in the + by an object storage. Large-scale log management and distributed dependency caching is under Instance object storage, set by the superadmins in the instance settings UI. {/if} @@ -1226,7 +1227,7 @@ on:click={() => { editWindmillLFSSettings() console.log('Saving S3 settings', s3ResourceSettings) - }}>Save S3 settingsSave storage settings
    {/if} diff --git a/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte b/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte index 645bc58c20..18b80ed3bf 100644 --- a/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte +++ b/frontend/src/routes/approve/[workspace]/[job]/[resume]/[hmac]/+page.svelte @@ -289,6 +289,7 @@
    diff --git a/frontend/src/routes/view_graph/+page.svelte b/frontend/src/routes/view_graph/+page.svelte index 18fbac237e..6844733561 100644 --- a/frontend/src/routes/view_graph/+page.svelte +++ b/frontend/src/routes/view_graph/+page.svelte @@ -3,15 +3,20 @@ import { decodeState } from '$lib/utils' let content = localStorage.getItem('svelvet') - const { modules, failureModule } = content + const { modules, failureModule, preprocessorModule } = content ? decodeState(content) - : { modules: [], failureModule: undefined } + : { modules: [], failureModule: undefined, preprocessorModule: undefined } - + DownloadDownload diff --git a/lsp/Pipfile b/lsp/Pipfile index 857cc0cac5..7d576e4659 100644 --- a/lsp/Pipfile +++ b/lsp/Pipfile @@ -4,8 +4,8 @@ verify_ssl = true name = "pypi" [packages] -wmill = ">=1.397.1" -wmill_pg = ">=1.397.1" +wmill = ">=1.398.1" +wmill_pg = ">=1.398.1" sendgrid = "*" mysql-connector-python = "*" pymongo = "*" diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index 83136aef4b..8c3394f511 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.397.1 + version: 1.398.1 title: OpenFlow Spec contact: name: Ruben Fiszel @@ -44,6 +44,8 @@ components: $ref: "#/components/schemas/FlowModule" failure_module: $ref: "#/components/schemas/FlowModule" + preprocessor_module: + $ref: "#/components/schemas/FlowModule" same_worker: type: boolean concurrent_limit: @@ -439,6 +441,9 @@ components: $ref: "#/components/schemas/FlowStatusModule" user_states: additionalProperties: true + preprocessor_module: + allOf: + - $ref: "#/components/schemas/FlowStatusModule" failure_module: allOf: - $ref: "#/components/schemas/FlowStatusModule" diff --git a/powershell-client/WindmillClient/WindmillClient.psd1 b/powershell-client/WindmillClient/WindmillClient.psd1 index a206a7362c..cd503aeb4e 100644 --- a/powershell-client/WindmillClient/WindmillClient.psd1 +++ b/powershell-client/WindmillClient/WindmillClient.psd1 @@ -12,7 +12,7 @@ RootModule = 'WindmillClient.psm1' # Version number of this module. -ModuleVersion = '1.397.1' +ModuleVersion = '1.398.1' # Supported PSEditions # CompatiblePSEditions = @() diff --git a/python-client/wmill/pyproject.toml b/python-client/wmill/pyproject.toml index 7065c6fd62..d13ae94461 100644 --- a/python-client/wmill/pyproject.toml +++ b/python-client/wmill/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill" -version = "1.397.1" +version = "1.398.1" description = "A client library for accessing Windmill server wrapping the Windmill client API" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/python-client/wmill_pg/pyproject.toml b/python-client/wmill_pg/pyproject.toml index 755ad89227..db56644f26 100644 --- a/python-client/wmill_pg/pyproject.toml +++ b/python-client/wmill_pg/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "wmill-pg" -version = "1.397.1" +version = "1.398.1" description = "An extension client for the wmill client library focused on pg" license = "Apache-2.0" homepage = "https://windmill.dev" diff --git a/typescript-client/client.ts b/typescript-client/client.ts index 2943aba5a5..c5256024a6 100644 --- a/typescript-client/client.ts +++ b/typescript-client/client.ts @@ -391,6 +391,7 @@ export async function setState(state: any): Promise { * Set the progress * Progress cannot go back and limited to 0% to 99% range * @param percent Progress to set in % + * @param jobId? Job to set progress for */ export async function setProgress(percent: number, jobId?: any): Promise { const workspace = getWorkspace(); @@ -412,7 +413,8 @@ export async function setProgress(percent: number, jobId?: any): Promise { id: jobId ?? getEnv("WM_JOB_ID") ?? "NO_JOB_ID", workspace, requestBody: { - percent, + // In case user inputs float, it should be converted to int + percent: Math.floor(percent), flow_job_id: (flowId == "") ? undefined : flowId, } }); @@ -420,6 +422,7 @@ export async function setProgress(percent: number, jobId?: any): Promise { /** * Get the progress + * @param jobId? Job to get progress from * @returns Optional clamped between 0 and 100 progress value */ export async function getProgress(jobId?: any): Promise { diff --git a/typescript-client/jsr.json b/typescript-client/jsr.json index ba057ecb63..03c201144d 100644 --- a/typescript-client/jsr.json +++ b/typescript-client/jsr.json @@ -1,6 +1,6 @@ { "name": "@windmill/windmill", - "version": "1.397.1", + "version": "1.398.1", "exports": "./src/index.ts", "publish": { "exclude": ["!src", "./s3Types.ts", "./client.ts"] diff --git a/typescript-client/package.json b/typescript-client/package.json index 256eb2e989..98426f0b17 100644 --- a/typescript-client/package.json +++ b/typescript-client/package.json @@ -1,7 +1,7 @@ { "name": "windmill-client", "description": "Windmill SDK client for browsers and Node.js", - "version": "1.397.1", + "version": "1.398.1", "author": "Ruben Fiszel", "license": "Apache 2.0", "devDependencies": { diff --git a/version.txt b/version.txt index 685ffb0bda..f8cd49121f 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -1.397.1 +1.398.1