diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index d0bfd53ae8..8cac56c35b 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -1,10 +1,8 @@ env: REGISTRY: ghcr.io - IMAGE_NAME: - ${{ github.event_name != 'pull_request' && github.repository || + IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.repository || 'windmill-labs/windmill-test' }} - DEV_SHA: - ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}', + DEV_SHA: ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}', github.event.number) }} name: Build windmill:main diff --git a/backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json b/backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json new file mode 100644 index 0000000000..a27bc6365a --- /dev/null +++ b/backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json @@ -0,0 +1,89 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_async", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "requires_auth", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "http_method: _", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441" +} diff --git a/backend/.sqlx/query-0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90.json b/backend/.sqlx/query-0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90.json deleted file mode 100644 index 4f6b896ad4..0000000000 --- a/backend/.sqlx/query-0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n WHERE id = $3", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Jsonb", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90" -} diff --git a/backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json b/backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json new file mode 100644 index 0000000000..00361e6e95 --- /dev/null +++ b/backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63" +} diff --git a/backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json b/backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json new file mode 100644 index 0000000000..4000bbb962 --- /dev/null +++ b/backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), ARRAY['step'], $2)\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f" +} diff --git a/backend/.sqlx/query-872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e.json b/backend/.sqlx/query-3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d.json similarity index 57% rename from backend/.sqlx/query-872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e.json rename to backend/.sqlx/query-3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d.json index cdf6a3f4fa..1d91cd4031 100644 --- a/backend/.sqlx/query-872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e.json +++ b/backend/.sqlx/query-3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return \n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", + "query": "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor \n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", "describe": { "columns": [ { @@ -15,8 +15,8 @@ }, { "ordinal": 2, - "name": "early_return", - "type_info": "Text" + "name": "has_preprocessor", + "type_info": "Bool" } ], "parameters": { @@ -31,5 +31,5 @@ null ] }, - "hash": "872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e" + "hash": "3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d" } diff --git a/backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json b/backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json new file mode 100644 index 0000000000..53a59847e7 --- /dev/null +++ b/backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE completed_job SET args = '{\"reason\":\"PREPROCESSOR_ARGS_ARE_DISCARDED\"}'::jsonb WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f" +} diff --git a/backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json b/backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json new file mode 100644 index 0000000000..b66edc6d73 --- /dev/null +++ b/backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json @@ -0,0 +1,38 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE http_trigger \n SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() \n WHERE workspace_id = $11 AND path = $12", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Varchar", + "Varchar", + "Bool", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea" +} diff --git a/backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json b/backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json new file mode 100644 index 0000000000..d954f2c26c --- /dev/null +++ b/backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json @@ -0,0 +1,108 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth\n FROM http_trigger\n WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "route_path_key", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "http_method: _", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + }, + { + "ordinal": 7, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "edited_at", + "type_info": "Timestamptz" + }, + { + "ordinal": 10, + "name": "extra_perms", + "type_info": "Jsonb" + }, + { + "ordinal": 11, + "name": "is_async", + "type_info": "Bool" + }, + { + "ordinal": 12, + "name": "requires_auth", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491" +} diff --git a/backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json b/backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json new file mode 100644 index 0000000000..ca0a922067 --- /dev/null +++ b/backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json @@ -0,0 +1,36 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() \n WHERE workspace_id = $9 AND path = $10", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bool", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Varchar", + "Varchar", + "Bool", + "Bool", + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0" +} diff --git a/backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json b/backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json similarity index 89% rename from backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json rename to backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json index ebd2e2535d..6bc0fb8a10 100644 --- a/backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json +++ b/backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2", + "query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where hash = $1 AND workspace_id = $2", "describe": { "columns": [ { @@ -82,6 +82,11 @@ "ordinal": 10, "name": "timeout", "type_info": "Int4" + }, + { + "ordinal": 11, + "name": "has_preprocessor", + "type_info": "Bool" } ], "parameters": { @@ -101,8 +106,9 @@ true, true, true, + true, true ] }, - "hash": "8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c" + "hash": "64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac" } diff --git a/backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json b/backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json new file mode 100644 index 0000000000..ec6143fe39 --- /dev/null +++ b/backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + ] + }, + "nullable": [ + null + ] + }, + "hash": "867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd" +} diff --git a/backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json b/backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json new file mode 100644 index 0000000000..f5c36ed27f --- /dev/null +++ b/backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json @@ -0,0 +1,17 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n WHERE id = $4", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368" +} diff --git a/backend/.sqlx/query-98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba.json b/backend/.sqlx/query-98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba.json new file mode 100644 index 0000000000..05cbaffc69 --- /dev/null +++ b/backend/.sqlx/query-98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba.json @@ -0,0 +1,41 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor\n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "tag", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "dedicated_worker", + "type_info": "Bool" + }, + { + "ordinal": 2, + "name": "early_return", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "has_preprocessor", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + true, + true, + null, + null + ] + }, + "hash": "98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba" +} diff --git a/backend/.sqlx/query-9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb.json b/backend/.sqlx/query-9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb.json new file mode 100644 index 0000000000..a0c5bd7642 --- /dev/null +++ b/backend/.sqlx/query-9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE path = $1 AND workspace_id = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb" +} diff --git a/backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json b/backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json new file mode 100644 index 0000000000..5de165462e --- /dev/null +++ b/backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Uuid", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb" +} diff --git a/backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json b/backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json new file mode 100644 index 0000000000..ae30d3bd98 --- /dev/null +++ b/backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef" +} diff --git a/backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json b/backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json new file mode 100644 index 0000000000..00453be45b --- /dev/null +++ b/backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n WHERE id = $3", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa" +} diff --git a/backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json b/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json similarity index 85% rename from backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json rename to backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json index 72ab4ffe26..71aaf4ba90 100644 --- a/backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json +++ b/backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)", + "query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31)", "describe": { "columns": [], "parameters": { @@ -72,10 +72,11 @@ "Varchar", "Bool", "Bool", - "Varchar" + "Varchar", + "Bool" ] }, "nullable": [] }, - "hash": "8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb" + "hash": "b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3" } diff --git a/backend/.sqlx/query-b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f.json b/backend/.sqlx/query-b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f.json new file mode 100644 index 0000000000..a730d4b588 --- /dev/null +++ b/backend/.sqlx/query-b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1 AND path = $2)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f" +} diff --git a/backend/.sqlx/query-b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9.json b/backend/.sqlx/query-b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9.json deleted file mode 100644 index 948e713ef6..0000000000 --- a/backend/.sqlx/query-b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n WHERE id = $4", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text", - "Jsonb", - "Jsonb", - "Uuid" - ] - }, - "nullable": [] - }, - "hash": "b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9" -} diff --git a/backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json b/backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json new file mode 100644 index 0000000000..7e9e9b84aa --- /dev/null +++ b/backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1)\n WHERE id = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Uuid" + ] + }, + "nullable": [] + }, + "hash": "c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772" +} diff --git a/backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json b/backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json new file mode 100644 index 0000000000..df94d893a6 --- /dev/null +++ b/backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json @@ -0,0 +1,37 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Varchar", + "Bool", + "Bool", + "Bool", + { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + }, + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8" +} diff --git a/backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json b/backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json new file mode 100644 index 0000000000..03b0c561e1 --- /dev/null +++ b/backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [] + }, + "hash": "dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188" +} diff --git a/backend/.sqlx/query-f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8.json b/backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json similarity index 74% rename from backend/.sqlx/query-f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8.json rename to backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json index a0a4a3770c..39c96b8989 100644 --- a/backend/.sqlx/query-f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8.json +++ b/backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1", + "query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1", "describe": { "columns": [ { @@ -24,5 +24,5 @@ null ] }, - "hash": "f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8" + "hash": "f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929" } diff --git a/backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json b/backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json similarity index 83% rename from backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json rename to backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json index 38c8521741..5e4f522147 100644 --- a/backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json +++ b/backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", + "query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", "describe": { "columns": [ { @@ -82,6 +82,11 @@ "ordinal": 10, "name": "timeout", "type_info": "Int4" + }, + { + "ordinal": 11, + "name": "has_preprocessor", + "type_info": "Bool" } ], "parameters": { @@ -101,8 +106,9 @@ true, true, true, + true, true ] }, - "hash": "e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99" + "hash": "fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a" } diff --git a/backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json b/backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json new file mode 100644 index 0000000000..65cb5c6633 --- /dev/null +++ b/backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json @@ -0,0 +1,87 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "script_path", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "is_flow", + "type_info": "Bool" + }, + { + "ordinal": 3, + "name": "route_path", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "workspace_id", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "is_async", + "type_info": "Bool" + }, + { + "ordinal": 6, + "name": "requires_auth", + "type_info": "Bool" + }, + { + "ordinal": 7, + "name": "edited_by", + "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "email", + "type_info": "Varchar" + }, + { + "ordinal": 9, + "name": "http_method: _", + "type_info": { + "Custom": { + "name": "http_method", + "kind": { + "Enum": [ + "get", + "post", + "put", + "delete", + "patch" + ] + } + } + } + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + "hash": "fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 52c7e5fb14..dbb8253306 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -10497,6 +10497,7 @@ dependencies = [ "lazy_static", "magic-crypt", "mail-parser", + "matchit", "mime_guess", "native-tls", "object_store", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index af33fc862b..b7e333fc8e 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -252,6 +252,7 @@ indexmap = { version = "2.2.5", features = ["serde"]} tokio-native-tls = "^0" openssl = "=0.10" mail-parser = "^0" +matchit = "=0.7.3" datafusion = "39.0.0" object_store = { version = "0.10.0", features = ["aws", "azure"] } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c2d0f0dd53..62ccf0ba0e 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -534f50aae2f89281304b3c3ba943d18afc07fad4 \ No newline at end of file +f506f5825d61e71420438389b88cdb0392bf993f \ No newline at end of file diff --git a/backend/migrations/20240917172503_job_routing.down.sql b/backend/migrations/20240917172503_job_routing.down.sql new file mode 100644 index 0000000000..569385aa5a --- /dev/null +++ b/backend/migrations/20240917172503_job_routing.down.sql @@ -0,0 +1,7 @@ +-- Add down migration script here +DROP TABLE http_trigger; +DROP TYPE http_method; + +ALTER TABLE script DROP COLUMN has_preprocessor; + +DROP FUNCTION prevent_route_path_change(); \ No newline at end of file diff --git a/backend/migrations/20240917172503_job_routing.up.sql b/backend/migrations/20240917172503_job_routing.up.sql new file mode 100644 index 0000000000..42dd41e199 --- /dev/null +++ b/backend/migrations/20240917172503_job_routing.up.sql @@ -0,0 +1,61 @@ +-- Add up migration script here +CREATE TYPE http_method AS ENUM ('get', 'post', 'put', 'delete', 'patch'); + +CREATE TABLE http_trigger ( + path VARCHAR(255) NOT NULL, + route_path VARCHAR(255) NOT NULL, + route_path_key VARCHAR(255) NOT NULL, + script_path VARCHAR(255) NOT NULL, + is_flow BOOLEAN NOT NULL, + workspace_id VARCHAR(50) NOT NULL, + edited_by VARCHAR(50) NOT NULL, + email VARCHAR(255) NOT NULL, + edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + extra_perms JSONB NOT NULL DEFAULT '{}', + is_async BOOLEAN NOT NULL DEFAULT FALSE, + requires_auth BOOLEAN NOT NULL DEFAULT FALSE, + http_method http_method NOT NULL, + PRIMARY KEY (path, workspace_id) +); + + +GRANT SELECT, UPDATE ON http_trigger TO windmill_user; +GRANT ALL ON http_trigger TO windmill_admin; + +ALTER TABLE http_trigger ENABLE ROW LEVEL SECURITY; + +CREATE POLICY see_folder_extra_perms_user_select ON http_trigger FOR SELECT TO windmill_user +USING (SPLIT_PART(http_trigger.path, '/', 1) = 'f' AND SPLIT_PART(http_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[])); +CREATE POLICY see_folder_extra_perms_user_update ON http_trigger FOR UPDATE TO windmill_user +USING (SPLIT_PART(http_trigger.path, '/', 1) = 'f' AND SPLIT_PART(http_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[])); + + +CREATE POLICY see_extra_perms_user_select ON http_trigger FOR SELECT TO windmill_user +USING (extra_perms ? CONCAT('u/', current_setting('session.user'))); +CREATE POLICY see_extra_perms_user_update ON http_trigger FOR UPDATE TO windmill_user +USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean); + +CREATE POLICY see_extra_perms_groups_select ON http_trigger FOR SELECT TO windmill_user +USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]); +CREATE POLICY see_extra_perms_groups_update ON http_trigger FOR UPDATE TO windmill_user +USING (exists( + SELECT key, value FROM jsonb_each_text(extra_perms) + WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]) + AND value::boolean)); + +CREATE OR REPLACE FUNCTION prevent_route_path_change() +RETURNS TRIGGER AS $$ +BEGIN + IF CURRENT_USER <> 'windmill_admin' AND NEW.route_path <> OLD.route_path THEN + RAISE EXCEPTION 'Modification of route_path is only allowed by admins'; + END IF; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +CREATE TRIGGER check_route_path_change +BEFORE UPDATE ON http_trigger +FOR EACH ROW +EXECUTE FUNCTION prevent_route_path_change(); + +ALTER TABLE script ADD COLUMN has_preprocessor BOOLEAN; diff --git a/backend/parsers/windmill-parser-bash/src/lib.rs b/backend/parsers/windmill-parser-bash/src/lib.rs index 515ef3822a..9b62ad3b0c 100644 --- a/backend/parsers/windmill-parser-bash/src/lib.rs +++ b/backend/parsers/windmill-parser-bash/src/lib.rs @@ -16,7 +16,13 @@ pub fn parse_bash_sig(code: &str) -> anyhow::Result { let parsed = parse_bash_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing bash script".to_string())) } @@ -26,7 +32,13 @@ pub fn parse_powershell_sig(code: &str) -> anyhow::Result { let parsed = parse_powershell_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing powershell script".to_string())) } @@ -173,7 +185,8 @@ non_required="${5:-}" oidx: None } ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-go/src/lib.rs b/backend/parsers/windmill-parser-go/src/lib.rs index c2564dbbd2..218a6dc3ec 100644 --- a/backend/parsers/windmill-parser-go/src/lib.rs +++ b/backend/parsers/windmill-parser-go/src/lib.rs @@ -42,6 +42,7 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result { star_kwargs: false, args, no_main_func: Some(false), + has_preprocessor: None, }) } else { Ok(MainArgSignature { @@ -49,6 +50,7 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result { star_kwargs: false, args: vec![], no_main_func: Some(true), + has_preprocessor: None, }) } } @@ -241,7 +243,8 @@ func main(x int, y string, z bool, l []string, o struct { Name string `json:"nam oidx: None }, ], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-graphql/src/lib.rs b/backend/parsers/windmill-parser-graphql/src/lib.rs index 060ae294d1..a900c3244d 100644 --- a/backend/parsers/windmill-parser-graphql/src/lib.rs +++ b/backend/parsers/windmill-parser-graphql/src/lib.rs @@ -15,7 +15,13 @@ pub fn parse_graphql_sig(code: &str) -> anyhow::Result { let parsed = parse_graphql_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -119,7 +125,8 @@ query($i: Int, $arr: [String]!, $wahoo: String = "wahoo") { oidx: None } ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-php/src/lib.rs b/backend/parsers/windmill-parser-php/src/lib.rs index 5adf1dc0d7..4c239c21c9 100644 --- a/backend/parsers/windmill-parser-php/src/lib.rs +++ b/backend/parsers/windmill-parser-php/src/lib.rs @@ -82,6 +82,7 @@ pub fn parse_php_signature( star_kwargs: false, args, no_main_func: Some(false), + has_preprocessor: None, }) } else { Ok(MainArgSignature { @@ -89,6 +90,7 @@ pub fn parse_php_signature( star_kwargs: false, args: vec![], no_main_func: Some(true), + has_preprocessor: None, }) } } @@ -159,7 +161,8 @@ function main(string $input1 = \"hey\", bool $input2 = false, int $input3 = 3, f oidx: None } ], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-py/src/lib.rs b/backend/parsers/windmill-parser-py/src/lib.rs index 3adde32185..5b42171478 100644 --- a/backend/parsers/windmill-parser-py/src/lib.rs +++ b/backend/parsers/windmill-parser-py/src/lib.rs @@ -63,9 +63,17 @@ pub fn parse_python_signature( ) -> anyhow::Result { let main_name = override_main.unwrap_or("main".to_string()); + let has_preprocessor = !filter_non_main(code, "preprocessor").is_empty(); + let filtered_code = filter_non_main(code, &main_name); if filtered_code.is_empty() { - return Err(anyhow::anyhow!("No main function found".to_string(),)); + return Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![], + no_main_func: Some(true), + has_preprocessor: Some(has_preprocessor), + }); } let ast = Suite::parse(&filtered_code, "main.py") .map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?; @@ -119,6 +127,7 @@ pub fn parse_python_signature( }) .collect(), no_main_func: Some(false), + has_preprocessor: Some(has_preprocessor), }) } else { Ok(MainArgSignature { @@ -126,6 +135,7 @@ pub fn parse_python_signature( star_kwargs: false, args: vec![], no_main_func: Some(true), + has_preprocessor: Some(has_preprocessor), }) } } @@ -325,6 +335,7 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt }, ], no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -389,6 +400,7 @@ def main(test1: str, } ], no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -448,6 +460,7 @@ def main(test1: str, } ], no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -491,6 +504,7 @@ def main(test1: Literal["foo", "bar"], test2: List[Literal["foo", "bar"]]): retu } ], no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -521,6 +535,59 @@ def main(test1: DynSelect_foo): return oidx: None }], no_main_func: Some(false), + has_preprocessor: Some(false) + } + ); + + Ok(()) + } + + #[test] + fn test_parse_python_sig_6() -> anyhow::Result<()> { + let code = r#" + +import os + +def hello(): return + +"#; + //println!("{}", serde_json::to_string()?); + assert_eq!( + parse_python_signature(code, None)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![], + no_main_func: Some(true), + has_preprocessor: Some(false) + } + ); + + Ok(()) + } + + #[test] + fn test_parse_python_sig_7() -> anyhow::Result<()> { + let code = r#" + +import os + +def preprocessor(): return + +def main(): return + + + +"#; + //println!("{}", serde_json::to_string()?); + assert_eq!( + parse_python_signature(code, None)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![], + no_main_func: Some(false), + has_preprocessor: Some(true) } ); diff --git a/backend/parsers/windmill-parser-rust/src/lib.rs b/backend/parsers/windmill-parser-rust/src/lib.rs index f9a2a5b677..5d4d626f4c 100644 --- a/backend/parsers/windmill-parser-rust/src/lib.rs +++ b/backend/parsers/windmill-parser-rust/src/lib.rs @@ -29,6 +29,7 @@ pub fn parse_rust_signature(code: &str) -> anyhow::Result { star_kwargs: false, args, no_main_func: Some(false), + has_preprocessor: None, }) } else { Ok(MainArgSignature { @@ -36,6 +37,7 @@ pub fn parse_rust_signature(code: &str) -> anyhow::Result { star_kwargs: false, args: vec![], no_main_func: Some(true), + has_preprocessor: None, }) } } @@ -455,10 +457,7 @@ fn main( assert_eq!(ret.args[2].typ, Typ::Str(None)); assert_eq!(ret.args[3].name, "my_string_vec"); - assert_eq!( - ret.args[3].otyp, - Some("Vec < String >".to_string()) - ); + assert_eq!(ret.args[3].otyp, Some("Vec < String >".to_string())); assert_eq!(ret.args[3].typ, Typ::List(Box::new(Typ::Str(None)))); } diff --git a/backend/parsers/windmill-parser-sql/src/lib.rs b/backend/parsers/windmill-parser-sql/src/lib.rs index fbb4ac5d40..dd6bc2b955 100644 --- a/backend/parsers/windmill-parser-sql/src/lib.rs +++ b/backend/parsers/windmill-parser-sql/src/lib.rs @@ -20,7 +20,13 @@ pub fn parse_mysql_sig(code: &str) -> anyhow::Result { let parsed = parse_mysql_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -30,7 +36,13 @@ pub fn parse_pgsql_sig(code: &str) -> anyhow::Result { let parsed = parse_pg_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -40,7 +52,13 @@ pub fn parse_bigquery_sig(code: &str) -> anyhow::Result { let parsed = parse_bigquery_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -50,7 +68,13 @@ pub fn parse_snowflake_sig(code: &str) -> anyhow::Result { let parsed = parse_snowflake_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -60,7 +84,13 @@ pub fn parse_mssql_sig(code: &str) -> anyhow::Result { let parsed = parse_mssql_file(&code)?; if let Some(x) = parsed { let args = x; - Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None }) + Ok(MainArgSignature { + star_args: false, + star_kwargs: false, + args, + no_main_func: None, + has_preprocessor: None, + }) } else { Err(anyhow!("Error parsing sql".to_string())) } @@ -570,7 +600,8 @@ SELECT * FROM table WHERE token=$1::TEXT AND image=$2::BIGINT oidx: Some(2), }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -618,7 +649,8 @@ SELECT $2::TEXT; oidx: Some(3), }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -744,7 +776,8 @@ SELECT ?, ?; oidx: None, }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -791,7 +824,8 @@ SELECT :param2; oidx: None, }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -830,7 +864,8 @@ SELECT @token; oidx: None, }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -877,7 +912,8 @@ SELECT ?; oidx: None, } ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); @@ -924,7 +960,8 @@ SELECT @p2; oidx: None, }, ], - no_main_func: None + no_main_func: None, + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-ts/src/lib.rs b/backend/parsers/windmill-parser-ts/src/lib.rs index 41ad759f0a..742a840802 100644 --- a/backend/parsers/windmill-parser-ts/src/lib.rs +++ b/backend/parsers/windmill-parser-ts/src/lib.rs @@ -159,6 +159,14 @@ pub fn parse_deno_signature( })? .body; + let has_preprocessor = ast.iter().any(|x| match x { + ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl { + decl: Decl::Fn(FnDecl { ident: Ident { sym, .. }, .. }), + .. + })) => &sym.to_string() == "preprocessor", + _ => false, + }); + let main_name = main_override.unwrap_or("main".to_string()); let params = ast.into_iter().find_map(|x| match x { ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl { @@ -178,6 +186,7 @@ pub fn parse_deno_signature( .map(|x| parse_param(x, &cm, skip_dflt, &mut c)) .collect::>>()?, no_main_func: Some(false), + has_preprocessor: Some(has_preprocessor), }; Ok(r) } else { @@ -186,6 +195,7 @@ pub fn parse_deno_signature( star_kwargs: false, args: vec![], no_main_func: Some(true), + has_preprocessor: Some(has_preprocessor), }) } } diff --git a/backend/parsers/windmill-parser-wasm/build-deno-pkgs.sh b/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh similarity index 64% rename from backend/parsers/windmill-parser-wasm/build-deno-pkgs.sh rename to backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh index 3ac0f9b424..7803fe22f0 100755 --- a/backend/parsers/windmill-parser-wasm/build-deno-pkgs.sh +++ b/backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh @@ -10,35 +10,41 @@ wasm-pack build --release --target web --out-dir $OUT_DIR --all-features \ OUT_DIR="pkg-ts" wasm-pack build --release --target web --out-dir $OUT_DIR --features "ts-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-ts"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-ts"/' $OUT_DIR/package.json # sql languages, graphql and bash/powershell, since they all use regex OUT_DIR="pkg-regex" wasm-pack build --release --target web --out-dir $OUT_DIR \ --features "sql-parser,graphql-parser,bash-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-regex"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-regex"/' $OUT_DIR/package.json # python OUT_DIR="pkg-py" wasm-pack build --release --target web --out-dir $OUT_DIR --features "py-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-py"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-py"/' $OUT_DIR/package.json # go OUT_DIR="pkg-go" wasm-pack build --release --target web --out-dir $OUT_DIR --features "go-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-go"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-go"/' $OUT_DIR/package.json # php OUT_DIR="pkg-php" wasm-pack build --release --target web --out-dir $OUT_DIR --features "php-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json # rust OUT_DIR="pkg-rust" wasm-pack build --release --target web --out-dir $OUT_DIR --features "rust-parser" \ -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort -sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json + +# ansible +OUT_DIR="pkg-yaml" +wasm-pack build --release --target web --out-dir $OUT_DIR --features "ansible-parser" \ + -Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort +sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-yaml"/' $OUT_DIR/package.json diff --git a/backend/parsers/windmill-parser-wasm/build.sh b/backend/parsers/windmill-parser-wasm/build.sh index 044f92d93b..b872e2c912 100755 --- a/backend/parsers/windmill-parser-wasm/build.sh +++ b/backend/parsers/windmill-parser-wasm/build.sh @@ -1,4 +1,4 @@ #!/bin/bash set -eou pipefail -deno task wasmbuild --out ../../../cli/wasm/ -p windmill-parser-wasm +deno task wasmbuild --out ../../../cli/wasm/ -p windmill-parser-wasm --all-features diff --git a/backend/parsers/windmill-parser-wasm/src/lib.rs b/backend/parsers/windmill-parser-wasm/src/lib.rs index c10f30c138..a5dd03ad91 100644 --- a/backend/parsers/windmill-parser-wasm/src/lib.rs +++ b/backend/parsers/windmill-parser-wasm/src/lib.rs @@ -17,8 +17,12 @@ fn wrap_sig(r: anyhow::Result) -> String { #[cfg(feature = "ts-parser")] #[wasm_bindgen] -pub fn parse_deno(code: &str) -> String { - wrap_sig(windmill_parser_ts::parse_deno_signature(code, false, None)) +pub fn parse_deno(code: &str, main_override: Option) -> String { + wrap_sig(windmill_parser_ts::parse_deno_signature( + code, + false, + main_override, + )) } #[cfg(feature = "ts-parser")] @@ -65,8 +69,11 @@ pub fn parse_go(code: &str) -> String { #[cfg(feature = "py-parser")] #[wasm_bindgen] -pub fn parse_python(code: &str) -> String { - wrap_sig(windmill_parser_py::parse_python_signature(code, None)) +pub fn parse_python(code: &str, main_override: Option) -> String { + wrap_sig(windmill_parser_py::parse_python_signature( + code, + main_override, + )) } #[cfg(feature = "sql-parser")] diff --git a/backend/parsers/windmill-parser-wasm/tests/wasm.rs b/backend/parsers/windmill-parser-wasm/tests/wasm.rs index 9f0feacbed..1c201ecd4e 100644 --- a/backend/parsers/windmill-parser-wasm/tests/wasm.rs +++ b/backend/parsers/windmill-parser-wasm/tests/wasm.rs @@ -139,7 +139,8 @@ export function main(test1?: string, test2: string = \"burkina\", oidx: None } ], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -215,7 +216,8 @@ export function main(test2 = \"burkina\", oidx: None } ], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -264,7 +266,8 @@ export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\", oidx: None } ], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -294,7 +297,8 @@ export function main(foo: (\"foo\" | \"bar\")[]) { has_default: false, oidx: None }], - no_main_func: Some(false) + no_main_func: Some(false), + has_preprocessor: Some(false) } ); @@ -434,6 +438,7 @@ Write-Output 'Testing...' } ], no_main_func: None, + has_preprocessor: None } ); diff --git a/backend/parsers/windmill-parser-yaml/src/lib.rs b/backend/parsers/windmill-parser-yaml/src/lib.rs index dd9e25e564..8c7068c54a 100644 --- a/backend/parsers/windmill-parser-yaml/src/lib.rs +++ b/backend/parsers/windmill-parser-yaml/src/lib.rs @@ -13,6 +13,7 @@ pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result anyhow::Result Typ { @@ -173,7 +180,12 @@ fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result anyhow::Result { if let Yaml::Hash(f) = yaml { - if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("resource".to_string())) - { + if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("resource".to_string())) { let target_path = f .get(&Yaml::String("target".to_string())) .and_then(|x| x.as_str()) @@ -262,7 +273,9 @@ fn parse_file_resource(yaml: &Yaml) -> anyhow::Result { ))?; return Ok(FileResource { resource_path: resource_path.clone(), target_path }); } - return Err(anyhow!("File resource should have a `resource` field, linking to a text file resource")); + return Err(anyhow!( + "File resource should have a `resource` field, linking to a text file resource" + )); } return Err(anyhow!("Invalid file resource: Should be a dictionary.")); } diff --git a/backend/parsers/windmill-parser/src/lib.rs b/backend/parsers/windmill-parser/src/lib.rs index c65b95d07c..6f793d0cc7 100644 --- a/backend/parsers/windmill-parser/src/lib.rs +++ b/backend/parsers/windmill-parser/src/lib.rs @@ -16,6 +16,7 @@ pub struct MainArgSignature { pub star_kwargs: bool, pub args: Vec, pub no_main_func: Option, + pub has_preprocessor: Option, } #[derive(Serialize, Clone, Debug, PartialEq)] @@ -153,7 +154,10 @@ mod test { #[test] fn test_mixed_delimiters() { - assert_eq!(to_snake_case("test-Case_with Spaces"), "test_case_with_spaces"); + assert_eq!( + to_snake_case("test-Case_with Spaces"), + "test_case_with_spaces" + ); } #[test] diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index ec4f4cd9f8..e92578b02f 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -3294,6 +3294,7 @@ async fn run_deployed_relative_imports( visible_to_runner_only: None, no_main_func: None, codebase: None, + has_preprocessor: None, }, ) .await @@ -3324,6 +3325,7 @@ async fn run_deployed_relative_imports( dedicated_worker: None, language, priority: None, + apply_preprocessor: false, }) .push(&db2) .await; diff --git a/backend/windmill-api/Cargo.toml b/backend/windmill-api/Cargo.toml index 8afb489378..fd8bedf0f9 100644 --- a/backend/windmill-api/Cargo.toml +++ b/backend/windmill-api/Cargo.toml @@ -94,6 +94,7 @@ object_store = { workspace = true, optional = true} openidconnect = { workspace = true, optional = true} url = { workspace = true, optional = true} jsonwebtoken = { workspace = true } +matchit.workspace = true pin-project.workspace = true crc.workspace = true diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 248ac7da12..999e66f48f 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -4182,6 +4182,11 @@ paths: in: query schema: type: integer + - name: skip_preprocessor + description: skip the preprocessor + in: query + schema: + type: boolean - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" @@ -5328,6 +5333,11 @@ paths: in: query schema: type: integer + - name: skip_preprocessor + description: skip the preprocessor + in: query + schema: + type: boolean - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/NewJobId" @@ -5436,6 +5446,11 @@ paths: in: query schema: type: integer + - name: skip_preprocessor + description: skip the preprocessor + in: query + schema: + type: boolean - $ref: "#/components/parameters/ParentJob" - $ref: "#/components/parameters/WorkerTag" - $ref: "#/components/parameters/CacheTtl" @@ -6822,6 +6837,190 @@ paths: "201": description: default error handler set + + /w/{workspace}/http_triggers/create: + post: + summary: create http trigger + operationId: createHttpTrigger + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: new http trigger + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/NewHttpTrigger" + responses: + "201": + description: http trigger created + content: + text/plain: + schema: + type: string + + /w/{workspace}/http_triggers/update/{path}: + post: + summary: update http trigger + operationId: updateHttpTrigger + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + requestBody: + description: updated trigger + required: true + content: + application/json: + schema: + $ref: "#/components/schemas/EditHttpTrigger" + responses: + "200": + description: http trigger updated + content: + text/plain: + schema: + type: string + + /w/{workspace}/http_triggers/delete/{path}: + delete: + summary: delete http trigger + operationId: deleteHttpTrigger + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: http trigger deleted + content: + text/plain: + schema: + type: string + + /w/{workspace}/http_triggers/get/{path}: + get: + summary: get http trigger + operationId: getHttpTrigger + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: http trigger deleted + content: + application/json: + schema: + $ref: "#/components/schemas/HttpTrigger" + + + /w/{workspace}/http_triggers/list: + get: + summary: list http triggers + operationId: listHttpTriggers + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + required: true + - $ref: "#/components/parameters/Page" + - $ref: "#/components/parameters/PerPage" + - name: path + description: filter by path + in: query + schema: + type: string + - name: is_flow + in: query + schema: + type: boolean + - name: path_start + in: query + schema: + type: string + responses: + "200": + description: http trigger list + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/HttpTrigger" + + + /w/{workspace}/http_triggers/exists/{path}: + get: + summary: does http trigger exists + operationId: existsHttpTrigger + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/Path" + responses: + "200": + description: http trigger exists + content: + application/json: + schema: + type: boolean + + /w/{workspace}/http_triggers/route_exists: + post: + summary: does route exists + operationId: existsRoute + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + requestBody: + description: route exists request + required: true + content: + application/json: + schema: + type: object + properties: + route_path: + type: string + http_method: + type: string + enum: ["get", "post", "put", "delete", "patch"] + required: + - kind + - route_path + - http_method + responses: + "200": + description: route exists + content: + application/json: + schema: + type: boolean + + /w/{workspace}/http_triggers/used: + get: + summary: whether http triggers are used + operationId: used + tags: + - http_trigger + parameters: + - $ref: "#/components/parameters/WorkspaceId" + responses: + "200": + description: whether http triggers are used + content: + application/json: + schema: + type: boolean + /groups/list: get: summary: list instance groups @@ -7648,6 +7847,7 @@ paths: folder, app, raw_app, + http_trigger, ] responses: "200": @@ -7684,6 +7884,7 @@ paths: folder, app, raw_app, + http_trigger, ] requestBody: description: acl to add @@ -7731,6 +7932,7 @@ paths: folder, app, raw_app, + http_trigger, ] requestBody: description: acl to add @@ -9421,6 +9623,8 @@ components: type: boolean codebase: type: string + has_preprocessor: + type: boolean required: - hash @@ -9438,6 +9642,7 @@ components: - kind - starred - no_main_func + - has_preprocessor NewScript: @@ -9519,6 +9724,8 @@ components: type: boolean codebase: type: string + has_preprocessor: + type: boolean required: - path - summary @@ -10319,6 +10526,9 @@ components: no_main_func: type: boolean nullable: true + has_preprocessor: + type: boolean + nullable: true required: - star_args - start_kwargs @@ -10326,6 +10536,7 @@ components: - type - error - no_main_func + - has_preprocessor Preview: type: object @@ -10715,6 +10926,122 @@ components: - is_flow - args + HttpTrigger: + type: object + properties: + path: + type: string + edited_by: + type: string + edited_at: + type: string + format: date-time + script_path: + type: string + route_path: + type: string + is_flow: + type: boolean + extra_perms: + type: object + additionalProperties: + type: boolean + email: + type: string + workspace_id: + type: string + http_method: + type: string + enum: + - get + - post + - put + - delete + - patch + is_async: + type: boolean + requires_auth: + type: boolean + + required: + - path + - edited_by + - edited_at + - script_path + - route_path + - extra_perms + - is_flow + - email + - workspace_id + - is_async + - requires_auth + - http_method + + NewHttpTrigger: + type: object + properties: + path: + type: string + script_path: + type: string + route_path: + type: string + is_flow: + type: boolean + http_method: + type: string + enum: + - get + - post + - put + - delete + - patch + is_async: + type: boolean + requires_auth: + type: boolean + + required: + - path + - script_path + - route_path + - is_flow + - is_async + - requires_auth + - http_method + + EditHttpTrigger: + type: object + properties: + path: + type: string + script_path: + type: string + route_path: + type: string + is_flow: + type: boolean + http_method: + type: string + enum: + - get + - post + - put + - delete + - patch + is_async: + type: boolean + requires_auth: + type: boolean + required: + - path + - script_path + - is_flow + - kind + - is_async + - requires_auth + - http_method + Group: type: object properties: diff --git a/backend/windmill-api/src/flows.rs b/backend/windmill-api/src/flows.rs index 2fedf3709c..6e74866203 100644 --- a/backend/windmill-api/src/flows.rs +++ b/backend/windmill-api/src/flows.rs @@ -1258,6 +1258,7 @@ mod tests { delete_after_use: None, continue_on_error: None, })), + preprocessor_module: None, same_worker: false, concurrent_limit: None, concurrency_time_window_s: None, @@ -1330,7 +1331,8 @@ mod tests { "expr": "previous.isEmpty()", "skip_if_stopped": false } - } + }, + "preprocessor_module": Option::::None }); assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect)); } diff --git a/backend/windmill-api/src/granular_acls.rs b/backend/windmill-api/src/granular_acls.rs index fb6d2e26f5..ad5793816b 100644 --- a/backend/windmill-api/src/granular_acls.rs +++ b/backend/windmill-api/src/granular_acls.rs @@ -23,8 +23,17 @@ use windmill_common::{ utils::{not_found_if_none, StripPath}, }; -const KINDS: [&str; 9] = [ - "script", "group_", "resource", "schedule", "variable", "flow", "folder", "app", "raw_app", +const KINDS: [&str; 10] = [ + "script", + "group_", + "resource", + "schedule", + "variable", + "flow", + "folder", + "app", + "raw_app", + "http_trigger", ]; pub fn workspaced_service() -> Router { diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs new file mode 100644 index 0000000000..b835c510c0 --- /dev/null +++ b/backend/windmill-api/src/http_triggers.rs @@ -0,0 +1,630 @@ +use axum::{ + extract::{Path, Query}, + response::IntoResponse, + routing::{any, delete, get, post}, + Extension, Json, Router, +}; +use http::{HeaderMap, StatusCode}; +use serde::{Deserialize, Serialize}; +use sql_builder::{bind::Bind, SqlBuilder}; +use sqlx::prelude::FromRow; +use std::collections::HashMap; +use windmill_audit::{audit_ee::audit_log, ActionKind}; +use windmill_common::{ + auth::fetch_authed_from_permissioned_as, + db::UserDB, + error::{self, JsonResult}, + users::username_to_permissioned_as, + utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, + worker::{to_raw_value, CLOUD_HOSTED}, +}; +use windmill_queue::PushArgsOwned; + +use crate::{ + db::{ApiAuthed, DB}, + jobs::{ + run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_flow_by_path_internal, + run_wait_result_script_by_path_internal, RunJobQuery, + }, + users::OptAuthed, +}; + +lazy_static::lazy_static! { + static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/:\w+").unwrap(); +} + +pub fn routes_global_service() -> Router { + Router::new().route("/*path", any(route_job)) +} + +pub fn workspaced_service() -> Router { + Router::new() + .route("/create", post(create_trigger)) + .route("/list", get(list_triggers)) + .route("/get/*path", get(get_trigger)) + .route("/update/*path", post(update_trigger)) + .route("/delete/*path", delete(delete_trigger)) + .route("/exists/*path", get(exists_trigger)) + .route("/used", get(used)) + .route("/route_exists", post(exists_route)) +} + +#[derive(Serialize, Deserialize, sqlx::Type)] +#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")] +#[serde(rename_all = "lowercase")] +enum HttpMethod { + Get, + Post, + Put, + Delete, + Patch, +} + +impl From for http::Method { + fn from(method: HttpMethod) -> Self { + match method { + HttpMethod::Get => http::Method::GET, + HttpMethod::Post => http::Method::POST, + HttpMethod::Put => http::Method::PUT, + HttpMethod::Delete => http::Method::DELETE, + HttpMethod::Patch => http::Method::PATCH, + } + } +} + +#[derive(Deserialize)] +struct NewTrigger { + path: String, + route_path: String, + script_path: String, + is_flow: bool, + is_async: bool, + requires_auth: bool, + http_method: HttpMethod, +} + +#[derive(FromRow, Serialize)] +struct Trigger { + workspace_id: String, + path: String, + route_path: String, + route_path_key: String, + script_path: String, + is_flow: bool, + edited_by: String, + email: String, + edited_at: chrono::DateTime, + extra_perms: serde_json::Value, + is_async: bool, + requires_auth: bool, + http_method: HttpMethod, +} + +#[derive(Deserialize)] +struct EditTrigger { + path: String, + route_path: Option, + script_path: String, + is_flow: bool, + is_async: bool, + requires_auth: bool, + http_method: HttpMethod, +} + +#[derive(Deserialize)] +pub struct ListTriggerQuery { + pub page: Option, + pub per_page: Option, + pub path: Option, + pub is_flow: Option, + pub path_start: Option, +} + +async fn list_triggers( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Query(lst): Query, +) -> error::JsonResult> { + let mut tx = user_db.begin(&authed).await?; + let (per_page, offset) = paginate(Pagination { per_page: lst.per_page, page: lst.page }); + let mut sqlb = SqlBuilder::select_from("http_trigger") + .field("*") + .order_by("edited_at", true) + .and_where("workspace_id = ?".bind(&w_id)) + .offset(offset) + .limit(per_page) + .clone(); + if let Some(path) = lst.path { + sqlb.and_where_eq("script_path", "?".bind(&path)); + } + if let Some(is_flow) = lst.is_flow { + sqlb.and_where_eq("is_flow", "?".bind(&is_flow)); + } + if let Some(path_start) = &lst.path_start { + sqlb.and_where_like_left("path", path_start); + } + let sql = sqlb + .sql() + .map_err(|e| error::Error::InternalErr(e.to_string()))?; + let rows = sqlx::query_as::<_, Trigger>(&sql) + .fetch_all(&mut *tx) + .await?; + tx.commit().await?; + + Ok(Json(rows)) +} + +async fn get_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> error::JsonResult { + let mut tx = user_db.begin(&authed).await?; + let path = path.to_path(); + let trigger = sqlx::query_as!( + Trigger, + r#"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as "http_method: _", edited_by, email, edited_at, extra_perms, is_async, requires_auth + FROM http_trigger + WHERE workspace_id = $1 AND path = $2"#, + w_id, + path, + ) + .fetch_optional(&mut *tx) + .await?; + tx.commit().await?; + + let trigger = not_found_if_none(trigger, "Trigger", path)?; + + Ok(Json(trigger)) +} + +async fn create_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path(w_id): Path, + Json(ct): Json, +) -> error::Result<(StatusCode, String)> { + require_admin(authed.is_admin, &authed.username)?; + + let route_path_key = ROUTE_PATH_KEY_RE.replace_all(ct.route_path.as_str(), ":key"); + + let mut tx = user_db.begin(&authed).await?; + sqlx::query!( + "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())", + w_id, + ct.path, + ct.route_path, + &route_path_key, + ct.script_path, + ct.is_flow, + ct.is_async, + ct.requires_auth, + ct.http_method as HttpMethod, + &authed.username, + &authed.email + ) + .execute(&mut *tx).await?; + + audit_log( + &mut *tx, + &authed, + "http_triggers.create", + ActionKind::Create, + &w_id, + Some(ct.path.as_str()), + None, + ) + .await?; + + tx.commit().await?; + + Ok((StatusCode::CREATED, format!("{}", ct.path))) +} + +async fn update_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, + Json(ct): Json, +) -> error::Result { + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + + if authed.is_admin { + if ct.route_path.is_none() { + return Err(error::Error::BadRequest( + "route_path is required".to_string(), + )); + } + + let route_path_key = + ROUTE_PATH_KEY_RE.replace_all(ct.route_path.as_ref().unwrap().as_str(), ":key"); + + sqlx::query!( + "UPDATE http_trigger + SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() + WHERE workspace_id = $11 AND path = $12", + ct.route_path, + &route_path_key, + ct.script_path, + ct.path, + ct.is_flow, + ct.http_method as HttpMethod, + &authed.username, + &authed.email, + ct.is_async, + ct.requires_auth, + w_id, + path, + ) + .execute(&mut *tx).await?; + } else { + sqlx::query!( + "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() + WHERE workspace_id = $9 AND path = $10", + ct.script_path, + ct.path, + ct.is_flow, + ct.http_method as HttpMethod, + &authed.username, + &authed.email, + ct.is_async, + ct.requires_auth, + w_id, + path, + ) + .execute(&mut *tx).await?; + } + + audit_log( + &mut *tx, + &authed, + "http_triggers.update", + ActionKind::Create, + &w_id, + Some(path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(path.to_string()) +} + +async fn delete_trigger( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> error::Result { + require_admin(authed.is_admin, &authed.username)?; + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + sqlx::query!( + "DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2", + w_id, + path, + ) + .execute(&mut *tx) + .await?; + + audit_log( + &mut *tx, + &authed, + "http_triggers.delete", + ActionKind::Delete, + &w_id, + Some(path), + None, + ) + .await?; + + tx.commit().await?; + + Ok(format!("HTTP trigger {path} deleted")) +} + +async fn used(Extension(db): Extension, Path(w_id): Path) -> JsonResult { + let used = sqlx::query_scalar!( + r#"SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1)"#, + w_id, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + Ok(Json(used)) +} + +async fn exists_trigger( + Extension(db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE path = $1 AND workspace_id = $2)", + path, + w_id, + ) + .fetch_one(&db) + .await? + .unwrap_or(false); + Ok(Json(exists)) +} + +#[derive(Deserialize)] +struct RouteExists { + route_path: String, + http_method: HttpMethod, +} + +async fn exists_route( + Extension(db): Extension, + Path(w_id): Path, + Json(RouteExists { route_path, http_method }): Json, +) -> JsonResult { + let route_path_key = ROUTE_PATH_KEY_RE.replace_all(route_path.as_str(), ":key"); + let exists = if *CLOUD_HOSTED { + sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3)", + &route_path_key, + w_id, + http_method as HttpMethod + ) + .fetch_one(&db) + .await? + .unwrap_or(false) + } else { + sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2)", + &route_path_key, + http_method as HttpMethod + ) + .fetch_one(&db) + .await? + .unwrap_or(false) + }; + Ok(Json(exists)) +} + +struct TriggerRoute { + path: String, + script_path: String, + is_flow: bool, + route_path: String, + workspace_id: String, + is_async: bool, + requires_auth: bool, + edited_by: String, + email: String, + http_method: HttpMethod, +} + +async fn fetch_api_authed( + username: String, + email: String, + w_id: &str, + db: &DB, + username_override: String, +) -> error::Result { + let permissioned_as = username_to_permissioned_as(username.as_str()); + let authed = + fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?; + Ok(ApiAuthed { + username: username, + email: email, + is_admin: authed.is_admin, + is_operator: authed.is_operator, + groups: authed.groups, + folders: authed.folders, + scopes: authed.scopes, + username_override: Some(username_override), + }) +} + +async fn get_http_route_trigger( + route_path: &str, + opt_authed: Option, + db: &DB, + user_db: UserDB, +) -> error::Result<(TriggerRoute, String, HashMap, ApiAuthed)> { + let (mut triggers, route_path) = if *CLOUD_HOSTED { + let mut splitted = route_path.split("/"); + let w_id = splitted.next().ok_or_else(|| { + error::Error::BadRequest("Missing workspace id in route path".to_string()) + })?; + let route_path = StripPath(splitted.collect::>().join("/")); + let triggers = sqlx::query_as!( + TriggerRoute, + r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger WHERE workspace_id = $1"#, + w_id + ) + .fetch_all(db) + .await?; + (triggers, route_path) + } else { + let triggers = sqlx::query_as!( + TriggerRoute, + r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger"#, + ) + .fetch_all(db) + .await?; + (triggers, StripPath(route_path.to_string())) + }; + + let mut router = matchit::Router::new(); + + for (idx, trigger) in triggers.iter().enumerate() { + let route_path = trigger.route_path.clone(); + router.insert(route_path.as_str(), idx).unwrap_or_else(|e| { + tracing::warn!( + "Failed to consider http trigger route {}: {:?}", + route_path, + e, + ); + }); + } + + let trigger_idx = router.at(route_path.0.as_str()).ok(); + + let matchit::Match { value: trigger_idx, params } = + not_found_if_none(trigger_idx, "Trigger", route_path.0.as_str())?; + + let trigger = triggers.remove(trigger_idx.to_owned()); + + let params: HashMap = params + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let username_override = if trigger.requires_auth { + if let Some(authed) = opt_authed { + // check that the user has access to the trigger + let mut tx = user_db.begin(&authed).await?; + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1 AND path = $2)", + trigger.workspace_id, + trigger.path + ) + .fetch_one(&mut *tx) + .await? + .unwrap_or(false); + tx.commit().await?; + if exists { + Some(authed.display_username().to_owned()) + } else { + return Err(error::Error::NotAuthorized("Unauthorized".to_string())); + } + } else { + return Err(error::Error::NotAuthorized( + "Requires authentication".to_string(), + )); + } + } else { + None + }; + + let authed = fetch_api_authed( + trigger.edited_by.clone(), + trigger.email.clone(), + &trigger.workspace_id, + &db, + username_override.unwrap_or("anonymous".to_string()), + ) + .await?; + + Ok((trigger, route_path.0, params, authed)) +} + +async fn route_job( + Extension(db): Extension, + Extension(user_db): Extension, + Extension(rsmq): Extension>, + Path(route_path): Path, + OptAuthed(opt_authed): OptAuthed, + Query(query): Query>, + method: http::Method, + headers: HeaderMap, + mut args: PushArgsOwned, +) -> impl IntoResponse { + let route_path = route_path.to_path(); + let (trigger, called_path, params, authed) = + match get_http_route_trigger(route_path, opt_authed, &db, user_db.clone()).await { + Ok(trigger) => trigger, + Err(e) => return e.into_response(), + }; + let headers = headers + .iter() + .map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string())) + .collect::>(); + let extra = args.extra.get_or_insert_with(HashMap::new); + extra.insert( + "wm_trigger".to_string(), + to_raw_value(&serde_json::json!({ + "kind": "http", + "http": { + "route": trigger.route_path, + "path": called_path, + "method": method.to_string().to_lowercase(), + "params": params, + "query": query, + "headers": headers + }, + })), + ); + let http_method = http::Method::from(trigger.http_method); + + if http_method != method { + return error::Error::BadRequest("Invalid HTTP method".to_string()).into_response(); + } + + let label_prefix = Some(format!( + "http-{}-{}-", + http_method.as_str().to_lowercase(), + trigger.route_path + )); + + let run_query = RunJobQuery::default(); + + if trigger.is_flow { + if trigger.is_async { + run_flow_by_path_inner( + authed, + db, + user_db, + rsmq, + trigger.workspace_id.clone(), + StripPath(trigger.script_path.to_owned()), + run_query, + args, + label_prefix, + ) + .await + .into_response() + } else { + run_wait_result_flow_by_path_internal( + db, + run_query, + StripPath(trigger.script_path.to_owned()), + authed, + rsmq, + user_db, + args, + trigger.workspace_id.clone(), + label_prefix, + ) + .await + .into_response() + } + } else { + if trigger.is_async { + run_script_by_path_inner( + authed, + db, + user_db, + rsmq, + trigger.workspace_id.clone(), + StripPath(trigger.script_path.to_owned()), + run_query, + args, + label_prefix, + ) + .await + .into_response() + } else { + run_wait_result_script_by_path_internal( + db, + run_query, + StripPath(trigger.script_path.to_owned()), + authed, + rsmq, + user_db, + trigger.workspace_id.clone(), + args, + label_prefix, + ) + .await + .into_response() + } + } +} diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index cec301ad05..e0c118c7b3 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -556,9 +556,10 @@ pub async fn get_path_tag_limits_cache_for_hash<'c, R: rsmq_async::RsmqConnectio Option, Option, Option, + Option, )> { let script = sqlx::query!( - "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2", + "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where hash = $1 AND workspace_id = $2", hash, w_id ) @@ -583,6 +584,7 @@ pub async fn get_path_tag_limits_cache_for_hash<'c, R: rsmq_async::RsmqConnectio script.priority, script.delete_after_use, script.timeout, + script.has_preprocessor, )) } @@ -1058,6 +1060,7 @@ pub struct RunJobQuery { pub tag: Option, pub timeout: Option, pub cache_ttl: Option, + pub skip_preprocessor: Option, } impl RunJobQuery { @@ -2784,14 +2787,18 @@ pub async fn run_flow_by_path_inner( let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); - let (tag, dedicated_worker) = sqlx::query!( - "SELECT tag, dedicated_worker from flow WHERE path = $1 and workspace_id = $2", + let (tag, dedicated_worker, has_preprocessor) = sqlx::query!( + "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor + FROM flow + LEFT JOIN flow_version + ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] + WHERE flow.path = $1 and flow.workspace_id = $2", flow_path, w_id ) .fetch_optional(&mut tx) .await? - .map(|x| (x.tag, x.dedicated_worker)) + .map(|x| (x.tag, x.dedicated_worker, x.has_preprocessor)) .ok_or_else(|| { Error::NotFound(format!( "flow not found at path {flow_path} in workspace {w_id}" @@ -2807,7 +2814,12 @@ pub async fn run_flow_by_path_inner( &db, tx, &w_id, - JobPayload::Flow { path: flow_path.to_string(), dedicated_worker }, + JobPayload::Flow { + path: flow_path.to_string(), + dedicated_worker, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), + }, PushArgs { args: &args.args, extra: args.extra }, &label_prefix .map(|x| x + authed.display_username()) @@ -2973,7 +2985,7 @@ pub async fn run_script_by_path_inner( let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); let (job_payload, tag, _delete_after_use, timeout) = - script_path_to_payload(script_path, &mut tx, &w_id).await?; + script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?; let scheduled_for = run_query.get_scheduled_for(&db).await?; let tag = run_query.tag.clone().or(tag); @@ -3049,7 +3061,15 @@ pub async fn run_workflow_as_code( None, run_query.timeout, ), - JobKind::Script => script_path_to_payload(job.script_path(), &mut tx, &w_id).await?, + JobKind::Script => { + script_path_to_payload( + job.script_path(), + &mut tx, + &w_id, + run_query.skip_preprocessor, + ) + .await? + } _ => return Err(anyhow::anyhow!("Not supported").into()), }; @@ -3451,7 +3471,7 @@ pub async fn run_wait_result_job_by_path_get( let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); let (job_payload, tag, delete_after_use, timeout) = - script_path_to_payload(script_path, &mut tx, &w_id).await?; + script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag).await?; @@ -3527,7 +3547,7 @@ pub async fn run_wait_result_flow_by_path_get( let args = PushArgsOwned { extra: Some(payload_args), args: HashMap::new() }; run_wait_result_flow_by_path_internal( - db, run_query, flow_path, authed, rsmq, user_db, args, w_id, + db, run_query, flow_path, authed, rsmq, user_db, args, w_id, None, ) .await } @@ -3553,11 +3573,12 @@ pub async fn run_wait_result_script_by_path( user_db, w_id, args, + None, ) .await } -async fn run_wait_result_script_by_path_internal( +pub async fn run_wait_result_script_by_path_internal( db: sqlx::Pool, run_query: RunJobQuery, script_path: StripPath, @@ -3566,6 +3587,7 @@ async fn run_wait_result_script_by_path_internal( user_db: UserDB, w_id: String, args: PushArgsOwned, + label_prefix: Option, ) -> error::Result { check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?; let script_path = script_path.to_path(); @@ -3574,7 +3596,7 @@ async fn run_wait_result_script_by_path_internal( let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into(); let (job_payload, tag, delete_after_use, timeout) = - script_path_to_payload(script_path, &mut tx, &w_id).await?; + script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?; let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag).await?; @@ -3587,7 +3609,9 @@ async fn run_wait_result_script_by_path_internal( &w_id, job_payload, PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), + &label_prefix + .map(|x| x + authed.display_username()) + .unwrap_or_else(|| authed.display_username().to_string()), &authed.email, username_to_permissioned_as(&authed.username), None, @@ -3644,6 +3668,7 @@ pub async fn run_wait_result_script_by_hash( priority, delete_after_use, timeout, + has_preprocessor, ) = get_path_tag_limits_cache_for_hash(&mut tx, &w_id, hash).await?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { cache_ttl = Some(run_query_cache_ttl); @@ -3669,6 +3694,8 @@ pub async fn run_wait_result_script_by_hash( language, dedicated_worker, priority, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), }, PushArgs { args: &args.args, extra: args.extra }, authed.display_username(), @@ -3712,12 +3739,12 @@ pub async fn run_wait_result_flow_by_path( check_license_key_valid().await?; run_wait_result_flow_by_path_internal( - db, run_query, flow_path, authed, rsmq, user_db, args, w_id, + db, run_query, flow_path, authed, rsmq, user_db, args, w_id, None, ) .await } -async fn run_wait_result_flow_by_path_internal( +pub async fn run_wait_result_flow_by_path_internal( db: sqlx::Pool, run_query: RunJobQuery, flow_path: StripPath, @@ -3726,6 +3753,7 @@ async fn run_wait_result_flow_by_path_internal( user_db: UserDB, args: PushArgsOwned, w_id: String, + label_prefix: Option, ) -> error::Result { check_queue_too_long(&db, run_query.queue_limit).await?; @@ -3736,8 +3764,8 @@ async fn run_wait_result_flow_by_path_internal( let scheduled_for = run_query.get_scheduled_for(&db).await?; - let (tag, dedicated_worker, early_return) = sqlx::query!( - "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return + let (tag, dedicated_worker, early_return, has_preprocessor) = sqlx::query!( + "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor FROM flow LEFT JOIN flow_version ON flow_version.id = flow.versions[array_upper(flow.versions, 1)] @@ -3747,7 +3775,7 @@ async fn run_wait_result_flow_by_path_internal( ) .fetch_optional(&mut tx) .await? - .map(|x| (x.tag, x.dedicated_worker, x.early_return)) + .map(|x| (x.tag, x.dedicated_worker, x.early_return, x.has_preprocessor)) .ok_or_else(|| { Error::NotFound(format!( "flow not found at path {flow_path} in workspace {w_id}" @@ -3763,9 +3791,16 @@ async fn run_wait_result_flow_by_path_internal( &db, tx, &w_id, - JobPayload::Flow { path: flow_path.to_string(), dedicated_worker }, + JobPayload::Flow { + path: flow_path.to_string(), + dedicated_worker, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), + }, PushArgs { args: &args.args, extra: args.extra }, - authed.display_username(), + &label_prefix + .map(|x| x + authed.display_username()) + .unwrap_or_else(|| authed.display_username().to_string()), &authed.email, username_to_permissioned_as(&authed.username), scheduled_for, @@ -4202,6 +4237,7 @@ async fn add_batch_jobs( _priority, _delete_after_use, timeout, + _, ) = get_latest_deployed_hash_for_path(&db, &w_id, &path).await?; ( Some(script_hash), @@ -4228,7 +4264,11 @@ async fn add_batch_jobs( JobPayload::RawFlow { value: fv.clone(), path: None, restarted_from: None } } else { if let Some(path) = batch_info.path.as_ref() { - JobPayload::Flow { path: path.to_string(), dedicated_worker: None } + JobPayload::Flow { + path: path.to_string(), + dedicated_worker: None, + apply_preprocessor: false, + } } else { Err(anyhow::anyhow!( "Path is required if no value is not provided" @@ -4450,6 +4490,7 @@ pub async fn run_job_by_hash_inner( priority, _delete_after_use, // not taken into account in async endpoints timeout, + has_preprocessor, ) = get_path_tag_limits_cache_for_hash(&mut tx, &w_id, hash).await?; check_scopes(&authed, || format!("run:script/{path}"))?; if let Some(run_query_cache_ttl) = run_query.cache_ttl { @@ -4475,6 +4516,8 @@ pub async fn run_job_by_hash_inner( language, dedicated_worker, priority, + apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), }, PushArgs { args: &args.args, extra: args.extra }, &label_prefix diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 4a37ad8f3e..2553596522 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -60,6 +60,7 @@ mod flows; mod folders; mod granular_acls; mod groups; +mod http_triggers; mod indexer_ee; mod inputs; mod integration; @@ -283,7 +284,8 @@ pub async fn run_server( ) .nest("/variables", variables::workspaced_service()) .nest("/workspaces", workspaces::workspaced_service()) - .nest("/oidc", oidc_ee::workspaced_service()), + .nest("/oidc", oidc_ee::workspaced_service()) + .nest("/http_triggers", http_triggers::workspaced_service()), ) .nest("/workspaces", workspaces::global_service()) .nest( @@ -336,7 +338,7 @@ pub async fn run_server( ) .nest( "/w/:workspace_id/capture_u", - capture::global_service().layer(cors), + capture::global_service().layer(cors.clone()), ) .nest( "/auth", @@ -346,6 +348,12 @@ pub async fn run_server( "/oauth", oauth2_ee::global_service().layer(Extension(Arc::clone(&sp_extension))), ) + .nest( + "/r", + http_triggers::routes_global_service() + .layer(from_extractor::()) + .layer(cors), + ) .route("/version", get(git_v)) .route("/uptodate", get(is_up_to_date)) .route("/ee_license", get(ee_license)) diff --git a/backend/windmill-api/src/scripts.rs b/backend/windmill-api/src/scripts.rs index 61e8c553ee..c912237e47 100644 --- a/backend/windmill-api/src/scripts.rs +++ b/backend/windmill-api/src/scripts.rs @@ -103,6 +103,8 @@ pub struct ScriptWDraft { pub visible_to_runner_only: Option, #[serde(skip_serializing_if = "Option::is_none")] pub no_main_func: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_preprocessor: Option, } pub fn global_service() -> Router { @@ -612,8 +614,8 @@ async fn create_script_internal<'c>( content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \ draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \ dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \ - delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)", + delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31)", &w_id, &hash.0, ns.path, @@ -643,7 +645,8 @@ async fn create_script_internal<'c>( ns.concurrency_key, ns.visible_to_runner_only, ns.no_main_func, - codebase + codebase, + ns.has_preprocessor, ) .execute(&mut tx) .await?; @@ -738,6 +741,7 @@ async fn create_script_internal<'c>( }, ); } + let permissioned_as = username_to_permissioned_as(&authed.username); if needs_lock_gen { let tag = if ns.dedicated_worker.is_some_and(|x| x) { @@ -878,7 +882,7 @@ async fn get_script_by_path_w_draft( let mut tx = user_db.begin(&authed).await?; let script_o = sqlx::query_as::<_, ScriptWDraft>( - "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func FROM script LEFT JOIN draft ON + "SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor FROM script LEFT JOIN draft ON script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script' WHERE script.path = $1 AND script.workspace_id = $2 \ AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \ diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index e32953722d..7306063856 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -2317,6 +2317,8 @@ struct ScriptMetadata { pub codebase: Option, #[serde(skip_serializing_if = "Option::is_none")] pub concurrency_key: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_preprocessor: Option, } pub fn is_none_or_false(val: &Option) -> bool { @@ -2602,6 +2604,7 @@ async fn tarball_workspace( no_main_func: script.no_main_func, codebase: script.codebase, concurrency_key: script.concurrency_key, + has_preprocessor: script.has_preprocessor, }; let metadata_str = serde_json::to_string_pretty(&metadata).unwrap(); archive diff --git a/backend/windmill-common/src/flow_status.rs b/backend/windmill-common/src/flow_status.rs index 946cd45bff..035cadecd8 100644 --- a/backend/windmill-common/src/flow_status.rs +++ b/backend/windmill-common/src/flow_status.rs @@ -29,6 +29,7 @@ pub struct FlowStatus { pub step: i32, pub modules: Vec, pub failure_module: Box, + pub preprocessor_module: Option, #[serde(skip_serializing_if = "HashMap::is_empty")] #[serde(default)] @@ -355,7 +356,11 @@ impl FlowStatusModule { impl FlowStatus { pub fn new(f: &FlowValue) -> Self { Self { - step: 0, + step: if f.preprocessor_module.is_some() { + -1 + } else { + 0 + }, approval_conditions: None, modules: f .modules @@ -372,6 +377,13 @@ impl FlowStatus { .unwrap_or_else(|| "failure".to_string()), }, }), + preprocessor_module: if f.preprocessor_module.is_some() { + Some(FlowStatusModule::WaitingForPriorSteps { + id: f.preprocessor_module.as_ref().unwrap().id.clone(), + }) + } else { + None + }, cleanup_module: FlowCleanupModule { flow_jobs_to_clean: vec![] }, retry: RetryStatus { fail_count: 0, failed_jobs: vec![] }, restarted_from: None, diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 3d64dc09da..b1ad85da99 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -102,6 +102,8 @@ pub struct FlowValue { #[serde(default)] pub failure_module: Option>, #[serde(default)] + pub preprocessor_module: Option>, + #[serde(default)] #[serde(skip_serializing_if = "is_default")] pub same_worker: bool, diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index e9c099a73c..f7cfe1c9ee 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -11,6 +11,8 @@ use uuid::Uuid; pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; +pub const PREPROCESSOR_FAKE_ENTRYPOINT: &str = "__WM_PREPROCESSOR"; + use crate::{ error::{self, to_anyhow, Error}, flow_status::{FlowStatus, RestartedFrom}, @@ -302,6 +304,7 @@ pub enum JobPayload { dedicated_worker: Option, language: ScriptLang, priority: Option, + apply_preprocessor: bool, }, Code(RawCode), Dependencies { @@ -331,6 +334,7 @@ pub enum JobPayload { Flow { path: String, dedicated_worker: Option, + apply_preprocessor: bool, }, RestartedFlow { completed_job_id: Uuid, @@ -383,6 +387,7 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre script_path: &str, db: E, w_id: &str, + skip_preprocessor: Option, ) -> error::Result<(JobPayload, Option, Option, Option)> { let (job_payload, tag, delete_after_use, script_timeout) = if script_path.starts_with("hub/") { ( @@ -404,6 +409,7 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre priority, delete_after_use, script_timeout, + has_preprocessor, ) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?; ( JobPayload::ScriptHash { @@ -416,6 +422,8 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre language, dedicated_worker, priority, + apply_preprocessor: !skip_preprocessor.unwrap_or(false) + && has_preprocessor.unwrap_or(false), }, tag, delete_after_use, @@ -473,7 +481,7 @@ pub async fn get_payload_tag_from_prefixed_path<'e, E: sqlx::Executor<'e, Databa w_id: &str, ) -> Result<(JobPayload, Option), Error> { let (payload, tag, _, _) = if path.starts_with("script/") { - script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id).await? + script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id, Some(true)).await? } else if path.starts_with("flow/") { let path = path.strip_prefix("flow/").unwrap().to_string(); let r = sqlx::query!( @@ -486,7 +494,12 @@ pub async fn get_payload_tag_from_prefixed_path<'e, E: sqlx::Executor<'e, Databa let (tag, dedicated_worker) = r .map(|x| (x.tag, x.dedicated_worker)) .unwrap_or_else(|| (None, None)); - (JobPayload::Flow { path, dedicated_worker }, tag, None, None) + ( + JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false }, + tag, + None, + None, + ) } else { return Err(Error::BadRequest(format!( "path must start with script/ or flow/ (got {})", diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index ecdc4cd218..48c31fbebb 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -271,9 +271,10 @@ pub async fn get_latest_deployed_hash_for_path<'e, E: sqlx::Executor<'e, Databas Option, Option, Option, + Option, )> { let r_o = sqlx::query!( - "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where path = $1 AND workspace_id = $2 AND + "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where path = $1 AND workspace_id = $2 AND created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)", script_path, @@ -296,6 +297,7 @@ pub async fn get_latest_deployed_hash_for_path<'e, E: sqlx::Executor<'e, Databas script.priority, script.delete_after_use, script.timeout, + script.has_preprocessor, )) } diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 5899b35575..8732e6d221 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -193,6 +193,8 @@ pub struct Script { pub no_main_func: Option, #[serde(skip_serializing_if = "Option::is_none")] pub codebase: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub has_preprocessor: Option, } #[derive(Serialize, sqlx::FromRow)] @@ -289,6 +291,7 @@ pub struct NewScript { pub visible_to_runner_only: Option, pub no_main_func: Option, pub codebase: Option, + pub has_preprocessor: Option, } fn lock_deserialize<'de, D>(deserializer: D) -> Result, D::Error> diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 8a6fdcbab7..a577ffe21e 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -52,6 +52,7 @@ use windmill_common::{ }, jobs::{ get_payload_tag_from_prefixed_path, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode, + ENTRYPOINT_OVERRIDE, PREPROCESSOR_FAKE_ENTRYPOINT, }, schedule::Schedule, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, @@ -1021,6 +1022,7 @@ pub async fn add_completed_job< .clone() .unwrap_or_else(|| ScriptLang::Deno), priority: queued_job.priority, + apply_preprocessor: false, }, queued_job .args @@ -2992,7 +2994,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( mut tx: PushIsolationLevel<'c, R>, workspace_id: &str, job_payload: JobPayload, - args: PushArgs<'d>, + mut args: PushArgs<'d>, user: &str, mut email: &str, mut permissioned_as: String, @@ -3211,21 +3213,38 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( language, dedicated_worker, priority, - } => ( - Some(hash.0), - Some(path), - None, - JobKind::Script, - None, - None, - Some(language), - custom_concurrency_key, - concurrent_limit, - concurrency_time_window_s, - cache_ttl, - dedicated_worker, - priority, - ), + apply_preprocessor, + } => { + let extra = args.extra.get_or_insert_with(HashMap::new); + if apply_preprocessor { + extra.insert( + ENTRYPOINT_OVERRIDE.to_string(), + to_raw_value(&PREPROCESSOR_FAKE_ENTRYPOINT), + ); + extra.entry("wm_trigger".to_string()).or_insert_with(|| { + to_raw_value(&serde_json::json!({ + "kind": "webhook", + })) + }); + } else { + extra.remove("wm_trigger"); + } + ( + Some(hash.0), + Some(path), + None, + JobKind::Script, + None, + None, + Some(language), + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + cache_ttl, + dedicated_worker, + priority, + ) + } JobPayload::ScriptHub { path } => { if path == "hub/7771/slack" || path == "hub/7836/slack" { permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string(); @@ -3404,9 +3423,13 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( branch_or_iteration_n: restarted_from_val.branch_or_iteration_n, }), user_states, + preprocessor_module: None, } } - _ => FlowStatus::new(&value), // this is a new flow being pushed, flow_status is set to flow_value + _ => { + value.preprocessor_module = None; + FlowStatus::new(&value) + } // this is a new flow being pushed, flow_status is set to flow_value }; ( None, @@ -3473,6 +3496,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( early_return: None, concurrency_key: custom_concurrency_key.clone(), priority: priority, + preprocessor_module: None, }; ( None, @@ -3490,7 +3514,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( priority, ) } - JobPayload::Flow { path, dedicated_worker } => { + JobPayload::Flow { path, dedicated_worker, apply_preprocessor } => { let value_json = fetch_scalar_isolated!( sqlx::query_as::<_, FlowRawValue>( "SELECT flow_version.value FROM flow @@ -3518,6 +3542,18 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( let custom_concurrency_key = value.concurrency_key.clone(); let concurrency_time_window_s = value.concurrency_time_window_s.clone(); let concurrent_limit = value.concurrent_limit.clone(); + + let extra = args.extra.get_or_insert_with(HashMap::new); + if !apply_preprocessor { + value.preprocessor_module = None; + extra.remove("wm_trigger"); + } else { + extra.entry("wm_trigger".to_string()).or_insert_with(|| { + to_raw_value(&serde_json::json!({ + "kind": "webhook", + })) + }); + } let status = Some(FlowStatus::new(&value)); ( None, @@ -3574,6 +3610,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>( branch_or_iteration_n, }), user_states, + preprocessor_module: None, }; ( None, diff --git a/backend/windmill-queue/src/schedule.rs b/backend/windmill-queue/src/schedule.rs index aacfa87c10..d39053c3c7 100644 --- a/backend/windmill-queue/src/schedule.rs +++ b/backend/windmill-queue/src/schedule.rs @@ -120,7 +120,11 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>( .map(|x| (x.tag, x.dedicated_worker)) .unwrap_or_else(|| (None, None)); ( - JobPayload::Flow { path: schedule.script_path.clone(), dedicated_worker }, + JobPayload::Flow { + path: schedule.script_path.clone(), + dedicated_worker, + apply_preprocessor: false, + }, tag, None, ) @@ -188,6 +192,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>( dedicated_worker, language, priority, + apply_preprocessor: false, }, if schedule.tag.as_ref().is_some_and(|x| x != "") { schedule.tag.clone() diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 8c55ab92c6..95ec6e3f9f 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -14,7 +14,8 @@ use crate::common::build_envs_map; use crate::{ common::{ create_args_and_out_file, get_main_override, get_reserved_variables, handle_child, - parse_npm_config, read_file_content, read_result, start_child_process, write_file_binary, + parse_npm_config, read_file, read_file_content, read_result, start_child_process, + write_file_binary, }, AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH, @@ -34,7 +35,7 @@ use windmill_common::variables; use windmill_common::{ error::{self, Result}, get_latest_hash_for_path, - jobs::QueuedJob, + jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, scripts::ScriptLang, worker::{exists_in_cache, get_annotation, save_cache, write_file}, DB, @@ -749,6 +750,7 @@ pub async fn handle_bun_job( worker_name: &str, envs: HashMap, shared_mount: &str, + new_args: &mut Option>>, ) -> error::Result> { let mut annotation = windmill_common::worker::get_annotation(inner_content); @@ -782,7 +784,16 @@ pub async fn handle_bun_job( if codebase.is_some() { annotation.nodejs_mode = true } - let main_override = get_main_override(job.args.as_ref()); + let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) { + Some(main_override) => { + if main_override == PREPROCESSOR_FAKE_ENTRYPOINT { + (None, true) + } else { + (Some(main_override), false) + } + } + None => (None, false), + }; #[cfg(not(feature = "enterprise"))] if annotation.nodejs_mode || annotation.npm_mode { @@ -937,7 +948,23 @@ pub async fn handle_bun_job( let args = windmill_parser_ts::parse_deno_signature(inner_content, true, main_override.clone())? .args; - let dates = args + + let pre_args = if apply_preprocessor { + Some( + windmill_parser_ts::parse_deno_signature( + inner_content, + true, + Some("preprocessor".to_string()), + )? + .args, + ) + } else { + None + }; + + let dates = pre_args + .as_ref() + .unwrap_or(&args) .iter() .enumerate() .filter_map(|(i, x)| { @@ -961,14 +988,34 @@ pub async fn handle_bun_job( "./main.ts" }; - let wrapper_content: String = format!( + let preprocessor = if let Some(pre_args) = pre_args { + let pre_spread = pre_args.into_iter().map(|x| x.name).join(","); + format!( + r#"if (Main.preprocessor === undefined || typeof Main.preprocessor !== 'function') {{ + throw new Error("preprocessor function is missing"); + }} + function preArgsObjToArr({{ {pre_spread} }}) {{ + return [ {pre_spread} ]; + }} + args = await Main.preprocessor(...preArgsObjToArr(args)); + const args_json = JSON.stringify(args ?? null, (key, value) => typeof value === 'undefined' ? null : value); + await fs.writeFile('args.json', args_json, {{ encoding: 'utf8' }})"# + ) + } else { + "".to_string() + }; + + let wrapper_content = format!( r#" import * as Main from "{main_import}"; import * as fs from "fs/promises"; -const args = await fs.readFile('args.json', {{ encoding: 'utf8' }}).then(JSON.parse) - .then(({{ {spread} }}) => [ {spread} ]) +let args = await fs.readFile('args.json', {{ encoding: 'utf8' }}).then(JSON.parse); + +function argsObjToArr({{ {spread} }}) {{ + return [ {spread} ]; +}} BigInt.prototype.toJSON = function () {{ return this.toString(); @@ -976,7 +1023,12 @@ BigInt.prototype.toJSON = function () {{ {dates} async function run() {{ - let res = await Main.{main_name}(...args); + {preprocessor} + const argsArr = argsObjToArr(args); + if (Main.{main_name} === undefined || typeof Main.{main_name} !== 'function') {{ + throw new Error("{main_name} function is missing"); + }} + let res = await Main.{main_name}(...argsArr); const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value); await fs.writeFile("result.json", res_json); process.exit(0); @@ -1309,6 +1361,23 @@ try {{ false, ) .await?; + + if apply_preprocessor { + let args = read_file(&format!("{job_dir}/args.json")) + .await + .map_err(|e| { + error::Error::InternalErr(format!( + "error while reading args from preprocessing: {e:#}" + )) + })?; + let args: HashMap> = + serde_json::from_str(args.get()).map_err(|e| { + error::Error::InternalErr(format!( + "error while deserializing args from preprocessing: {e:#}" + )) + })?; + *new_args = Some(args.clone()); + } read_result(job_dir).await } diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index d60dfd977a..4df77dc21f 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -107,7 +107,7 @@ pub async fn create_args_and_out_file( job_dir: &str, db: &Pool, ) -> Result<(), Error> { - if let Some(args) = &job.args { + if let Some(args) = job.args.as_ref() { if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, db).await? { write_file( job_dir, @@ -380,8 +380,8 @@ pub async fn read_and_check_file(path: &str) -> error::Result> { check_result_too_big(content.len())?; - let raw_value: Box = serde_json::from_str(&content) - .map_err(|e| anyhow!("{} is not valid json: {}", path, e))?; + let raw_value: Box = + serde_json::from_str(&content).map_err(|e| anyhow!("{} is not valid json: {}", path, e))?; Ok(raw_value) } diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 4b139a63d1..2eece2a36f 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -8,13 +8,15 @@ use windmill_queue::{append_logs, CanceledBy}; use crate::{ common::{ create_args_and_out_file, get_main_override, get_reserved_variables, handle_child, - parse_npm_config, read_result, start_child_process, + parse_npm_config, read_file, read_result, start_child_process, }, AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV, NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV, }; use tokio::{fs::File, io::AsyncReadExt, process::Command}; -use windmill_common::{error::Result, worker::write_file, BASE_URL}; +use windmill_common::{ + error::Result, jobs::PREPROCESSOR_FAKE_ENTRYPOINT, worker::write_file, BASE_URL, +}; use windmill_common::{ error::{self}, jobs::QueuedJob, @@ -170,12 +172,22 @@ pub async fn handle_deno_job( base_internal_url: &str, worker_name: &str, envs: HashMap, + new_args: &mut Option>>, ) -> error::Result> { // let mut start = Instant::now(); let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string(); append_logs(&job.id, &job.workspace_id, logs1, db).await; - let main_override = get_main_override(job.args.as_ref()); + let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) { + Some(main_override) => { + if main_override == PREPROCESSOR_FAKE_ENTRYPOINT { + (None, true) + } else { + (Some(main_override), false) + } + } + None => (None, false), + }; write_file(job_dir, "main.ts", inner_content)?; @@ -184,7 +196,23 @@ pub async fn handle_deno_job( let args = windmill_parser_ts::parse_deno_signature(inner_content, true, main_override.clone())? .args; - let dates = args + + let pre_args = if apply_preprocessor { + Some( + windmill_parser_ts::parse_deno_signature( + inner_content, + true, + Some("preprocessor".to_string()), + )? + .args, + ) + } else { + None + }; + + let dates = pre_args + .as_ref() + .unwrap_or(&args) .iter() .enumerate() .filter_map(|(i, x)| { @@ -200,13 +228,37 @@ pub async fn handle_deno_job( let spread = args.into_iter().map(|x| x.name).join(","); let main_name = main_override.unwrap_or("main".to_string()); // logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str()); + let (preprocessor_import, preprocessor) = if let Some(pre_args) = pre_args { + let pre_spread = pre_args.into_iter().map(|x| x.name).join(","); + ( + r#"import { preprocessor } from "./main.ts";"#.to_string(), + format!( + r#"if (preprocessor === undefined || typeof preprocessor !== 'function') {{ + throw new Error("preprocessor function is missing"); + }} + function preArgsObjToArr({{ {pre_spread} }}) {{ + return [ {pre_spread} ]; + }} + args = await preprocessor(...preArgsObjToArr(args)); + const args_json = JSON.stringify(args ?? null, (key, value) => typeof value === 'undefined' ? null : value); + await Deno.writeTextFile("args.json", args_json);"# + ), + ) + } else { + ("".to_string(), "".to_string()) + }; + let wrapper_content: String = format!( r#" import {{ {main_name} }} from "./main.ts"; +{preprocessor_import} -const args = await Deno.readTextFile("args.json") - .then(JSON.parse) - .then(({{ {spread} }}) => [ {spread} ]) +let args = await Deno.readTextFile("args.json") + .then(JSON.parse); + +function argsObjToArr({{ {spread} }}) {{ + return [ {spread} ]; +}} BigInt.prototype.toJSON = function () {{ return this.toString(); @@ -214,7 +266,12 @@ BigInt.prototype.toJSON = function () {{ {dates} async function run() {{ - let res: any = await {main_name}(...args); + {preprocessor} + const argsArr = argsObjToArr(args); + if ({main_name} === undefined || typeof {main_name} !== 'function') {{ + throw new Error("{main_name} function is missing"); + }} + let res: any = await {main_name}(...argsArr); const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value); await Deno.writeTextFile("result.json", res_json); Deno.exit(0); @@ -346,6 +403,22 @@ try {{ { tracing::error!("failed to remove deno gen tmp cache dir: {}", e); } + if apply_preprocessor { + let args = read_file(&format!("{job_dir}/args.json")) + .await + .map_err(|e| { + error::Error::InternalErr(format!( + "error while reading args from preprocessing: {e:#}" + )) + })?; + let args: HashMap> = + serde_json::from_str(args.get()).map_err(|e| { + error::Error::InternalErr(format!( + "error while deserializing args from preprocessing: {e:#}" + )) + })?; + *new_args = Some(args.clone()); + } read_result(job_dir).await } diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 36de523eb3..51584eb037 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -14,7 +14,7 @@ use uuid::Uuid; use windmill_common::ee::{get_license_plan, LicensePlan}; use windmill_common::{ error::{self, Error}, - jobs::QueuedJob, + jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, utils::calculate_hash, worker::{write_file, WORKER_CONFIG}, DB, @@ -56,7 +56,7 @@ use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS; use crate::{ common::{ create_args_and_out_file, get_main_override, get_reserved_variables, handle_child, - read_result, start_child_process, + read_file, read_result, start_child_process, }, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, @@ -246,6 +246,7 @@ pub async fn handle_python_job( shared_mount: &str, base_internal_url: &str, envs: HashMap, + new_args: &mut Option>>, ) -> windmill_common::error::Result> { let script_path = crate::common::use_flow_root_path(job.script_path()); let additional_python_paths = handle_python_deps( @@ -281,10 +282,39 @@ pub async fn handle_python_job( transforms, spread, main_name, - ) = prepare_wrapper(job_dir, inner_content, &script_path, job.args.as_ref()).await?; + pre_spread, + ) = prepare_wrapper( + job_dir, + inner_content, + &script_path, + job.args.as_ref(), + false, + ) + .await?; + + let apply_preprocessor = pre_spread.is_some(); create_args_and_out_file(&client, job, job_dir, db).await?; + let preprocessor = if let Some(pre_spread) = pre_spread { + format!( + r#"if inner_script.preprocessor is None or not callable(inner_script.preprocessor): + raise ValueError("preprocessor function is missing") + else: + pre_args = {{}} + {pre_spread} + for k, v in list(pre_args.items()): + if v == '': + del pre_args[k] + kwargs = inner_script.preprocessor(**pre_args) + kwrags_json = res_to_json(kwargs) + with open("args.json", 'w') as f: + f.write(kwrags_json)"# + ) + } else { + "".to_string() + }; + let os_main_override = if let Some(main_override) = main_name.as_ref() { format!("os.environ[\"MAIN_OVERRIDE\"] = \"{main_override}\"\n") } else { @@ -308,10 +338,6 @@ with open("args.json") as f: kwargs = json.load(f, strict=False) args = {{}} {transforms} -{spread} -for k, v in list(args.items()): - if v == '': - del args[k] def to_b_64(v: bytes): import base64 @@ -322,8 +348,7 @@ replace_nan = re.compile(r'(?:\bNaN\b|\\*\\u0000)') result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json") -try: - res = inner_script.{main_override}(**args) +def res_to_json(res): typ = type(res) if typ.__name__ == 'DataFrame': if typ.__module__ == 'pandas.core.frame': @@ -336,7 +361,18 @@ try: for k, v in res.items(): if type(v).__name__ == 'bytes': res[k] = to_b_64(v) - res_json = re.sub(replace_nan, ' null ', json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')) + return re.sub(replace_nan, ' null ', json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')) + +try: + {preprocessor} + {spread} + for k, v in list(args.items()): + if v == '': + del args[k] + if inner_script.{main_override} is None or not callable(inner_script.{main_override}): + raise ValueError("{main_override} function is missing") + res = inner_script.{main_override}(**args) + res_json = res_to_json(res) with open(result_json, 'w') as f: f.write(res_json) except BaseException as e: @@ -451,6 +487,24 @@ mount {{ false, ) .await?; + + if apply_preprocessor { + let args = read_file(&format!("{job_dir}/args.json")) + .await + .map_err(|e| { + error::Error::InternalErr(format!( + "error while reading args from preprocessing: {e:#}" + )) + })?; + let args: HashMap> = + serde_json::from_str(args.get()).map_err(|e| { + error::Error::InternalErr(format!( + "error while deserializing args from preprocessing: {e:#}" + )) + })?; + *new_args = Some(args.clone()); + } + read_result(job_dir).await } @@ -459,6 +513,7 @@ async fn prepare_wrapper( inner_content: &str, script_path: &str, args: Option<&Json>>>, + skip_preprocessor: bool, ) -> error::Result<( &'static str, &'static str, @@ -469,8 +524,18 @@ async fn prepare_wrapper( String, String, Option, + Option, )> { - let main_override = get_main_override(args); + let (main_override, apply_preprocessor) = match get_main_override(args) { + Some(main_override) => { + if !skip_preprocessor && main_override == PREPROCESSOR_FAKE_ENTRYPOINT { + (None, true) + } else { + (Some(main_override), false) + } + } + None => (None, false), + }; let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content); @@ -511,7 +576,20 @@ async fn prepare_wrapper( } let sig = windmill_parser_py::parse_python_signature(inner_content, main_override.clone())?; - let transforms = sig + + let pre_sig = if apply_preprocessor { + Some(windmill_parser_py::parse_python_signature( + inner_content, + Some("preprocessor".to_string()), + )?) + } else { + None + }; + + // transforms should be applied based on the signature of the first function called + let init_sig = pre_sig.as_ref().unwrap_or(&sig); + + let transforms = init_sig .args .iter() .map(|x| match x.typ { @@ -539,7 +617,7 @@ async fn prepare_wrapper( } else { "" }; - let import_base64 = if sig + let import_base64 = if init_sig .args .iter() .any(|x| x.typ == windmill_parser::Typ::Bytes) @@ -548,7 +626,7 @@ async fn prepare_wrapper( } else { "" }; - let import_datetime = if sig + let import_datetime = if init_sig .args .iter() .any(|x| x.typ == windmill_parser::Typ::Datetime) @@ -569,12 +647,38 @@ async fn prepare_wrapper( } else { format!( r#"args["{name}"] = kwargs.get("{name}") -if args["{name}"] is None: - del args["{name}"]"# + if args["{name}"] is None: + del args["{name}"]"# ) } }) - .join("\n") + .join("\n ") + }; + + let pre_spread = if let Some(pre_sig) = pre_sig { + let spread = if pre_sig.star_kwargs { + "pre_args = kwargs".to_string() + } else { + pre_sig + .args + .into_iter() + .map(|x| { + let name = &x.name; + if x.default.is_none() { + format!("pre_args[\"{name}\"] = kwargs.get(\"{name}\")") + } else { + format!( + r#"pre_args["{name}"] = kwargs.get("{name}") + if pre_args["{name}"] is None: + del pre_args["{name}"]"# + ) + } + }) + .join("\n ") + }; + Some(spread) + } else { + None }; let module_dir_dot = dirs.replace("/", ".").replace("-", "_"); @@ -589,6 +693,7 @@ if args["{name}"] is None: transforms, spread, main_override, + pre_spread, )) } @@ -1082,7 +1187,8 @@ pub async fn start_worker( transforms, spread, _, - ) = prepare_wrapper(job_dir, inner_content, script_path, _args.as_ref()).await?; + _, + ) = prepare_wrapper(job_dir, inner_content, script_path, _args.as_ref(), true).await?; { let indented_transforms = transforms @@ -1090,11 +1196,6 @@ pub async fn start_worker( .map(|x| format!(" {}", x)) .collect::>() .join("\n"); - let indented_spread = spread - .lines() - .map(|x| format!(" {}", x)) - .collect::>() - .join("\n"); let wrapper_content: String = format!( r#" @@ -1122,7 +1223,7 @@ for line in sys.stdin: kwargs = json.loads(line, strict=False) args = {{}} {indented_transforms} -{indented_spread} + {spread} for k, v in list(args.items()): if v == '': del args[k] diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index e749f28d3d..9aaa4f2799 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -1,6 +1,6 @@ use serde::Serialize; use sqlx::{types::Json, Pool, Postgres}; -use std::sync::Arc; +use std::{collections::HashMap, sync::Arc}; use uuid::Uuid; @@ -53,26 +53,49 @@ pub async fn process_result( cached_res_path: Option, token: String, column_order: Option>, + new_args: Option>>, db: &DB, ) -> error::Result<()> { match result { Ok(r) => { - let job = if let Some(column_order) = column_order { - let mut job_with_column_order = (*job).clone(); - match job_with_column_order.flow_status { - Some(_) => { - tracing::warn!("flow_status was expected to be none"); - } - None => { - job_with_column_order.flow_status = - Some(sqlx::types::Json(to_raw_value(&serde_json::json!({ - "_metadata": { - "column_order": column_order - } - })))); + let job = if column_order.is_some() || new_args.is_some() { + let mut updated_job = (*job).clone(); + if let Some(column_order) = column_order { + match updated_job.flow_status { + Some(_) => { + tracing::warn!("flow_status was expected to be none"); + } + None => { + updated_job.flow_status = + Some(sqlx::types::Json(to_raw_value(&serde_json::json!({ + "_metadata": { + "column_order": column_order + } + })))); + } } } - Arc::new(job_with_column_order) + if let Some(new_args) = new_args { + match updated_job.flow_status { + Some(_) => { + tracing::warn!("flow_status was expected to be none"); + } + None => { + // TODO save original args somewhere + // if let Some(args) = updated_job.args.as_mut() { + // args.0.remove(ENTRYPOINT_OVERRIDE); + // } + updated_job.flow_status = + Some(sqlx::types::Json(to_raw_value(&serde_json::json!({ + "_metadata": { + "preprocessed_args": true + } + })))); + } + } + updated_job.args = Some(Json(new_args)); + } + Arc::new(updated_job) } else { job }; diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index ed8e432194..1e5ca26024 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -85,9 +85,9 @@ use rand::Rng; use crate::{ ansible_executor::handle_ansible_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{ build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, - NO_LOGS_AT_ALL, SLOW_LOGS, - }, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{handle_job_error, handle_receive_completed_job, process_result}, rust_executor::handle_rust_job, worker_flow::{ - handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, + NO_LOGS_AT_ALL, SLOW_LOGS, + }, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, handle_job_error, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{handle_receive_completed_job, process_result}, rust_executor::handle_rust_job, worker_flow::{ + handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, Step, }, worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, } @@ -2050,7 +2050,7 @@ async fn handle_queued_job( } #[cfg(any(not(feature = "enterprise"), feature = "sqlx"))] - if job.created_by.starts_with("email-trigger-") { + if job.created_by.starts_with("email-") { let daily_count = sqlx::query!( "SELECT value FROM metrics WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day' ORDER BY created_at DESC LIMIT 1" ).fetch_optional(db).await?.map(|x| serde_json::from_value::(x.value).unwrap_or(1)); @@ -2087,7 +2087,7 @@ async fn handle_queued_job( ) .await?; - r + Some(r) } else { if let Some(parent_job) = job.parent_job { if let Err(e) = sqlx::query_scalar!( @@ -2139,7 +2139,11 @@ async fn handle_queued_job( .await .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e:#}")))? .ok_or_else(|| Error::InternalErr(format!("Expected script_path")))?; - let step = step.unwrap_or(-1); + let step = match step.unwrap() { + Step::Step(i) => i.to_string(), + Step::PreprocessorStep => "preprocessor".to_string(), + Step::FailureStep => "failure".to_string(), + }; Some(format!( "{flow_path}/cache/{version_hash}/{step}/{args_hash}" )) @@ -2239,6 +2243,7 @@ async fn handle_queued_job( append_logs(&job.id, &job.workspace_id, logs, db).await; let mut column_order: Option> = None; + let mut new_args: Option>> = None; let result = match job.job_kind { JobKind::Dependencies => { handle_dependency_job( @@ -2306,6 +2311,7 @@ async fn handle_queued_job( base_internal_url, worker_name, &mut column_order, + &mut new_args, ) .await; *worker_code_execution_metric += metric_timer.elapsed().as_secs_f32(); @@ -2337,6 +2343,7 @@ async fn handle_queued_job( cached_res_path, client.get_token().await, column_order, + new_args, db, ) .await?; @@ -2432,7 +2439,7 @@ pub async fn get_script_content_by_hash( Option, Option, Option>, - Option, + Option ), >( "SELECT content, lock, language, envs, codebase LIKE '%.tar' as codebase FROM script WHERE hash = $1 AND workspace_id = $2", @@ -2473,6 +2480,7 @@ async fn handle_code_execution_job( base_internal_url: &str, worker_name: &str, column_order: &mut Option>, + new_args: &mut Option>>, ) -> error::Result> { let ContentReqLangEnvs { content: inner_content, @@ -2705,6 +2713,7 @@ mount {{ &shared_mount, base_internal_url, envs, + new_args, ) .await } @@ -2721,6 +2730,7 @@ mount {{ base_internal_url, worker_name, envs, + new_args, ) .await } @@ -2739,6 +2749,7 @@ mount {{ worker_name, envs, &shared_mount, + new_args, ) .await } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 9064cf1579..8c1161cc3e 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -37,7 +37,7 @@ use windmill_common::flow_status::{ use windmill_common::flows::add_virtual_items_if_necessary; use windmill_common::jobs::{ script_hash_to_tag_and_limits, script_path_to_payload, BranchResults, JobPayload, QueuedJob, - RawCode, + RawCode, ENTRYPOINT_OVERRIDE, }; use windmill_common::worker::to_raw_value; use windmill_common::{ @@ -225,14 +225,22 @@ pub async fn update_flow_status_after_job_completion_internal< ))) })?; - let module_index = usize::try_from(old_status.step).ok(); + let module_step = Step::from_i32_and_len(old_status.step, old_status.modules.len()); - let module_status = module_index - .and_then(|i| old_status.modules.get(i)) - .unwrap_or(&old_status.failure_module.module_status); + let module_status = match module_step { + Step::PreprocessorStep => old_status + .preprocessor_module + .as_ref() + .ok_or_else(|| Error::InternalErr(format!("preprocessor module not found")))?, + Step::FailureStep => &old_status.failure_module.module_status, + Step::Step(i) => old_status + .modules + .get(i as usize) + .ok_or_else(|| Error::InternalErr(format!("module {i} not found")))?, + }; // tracing::debug!( - // "UPDATE FLOW STATUS 2: {module_index:#?} {module_status:#?} {old_status:#?} " + // "UPDATE FLOW STATUS 2: {module_step:#?} {module_status:#?} {old_status:#?} " // ); let (is_loop, skip_loop_failures, parallelism) = if matches!( @@ -262,14 +270,14 @@ pub async fn update_flow_status_after_job_completion_internal< let flow_job = get_queued_job(&flow, w_id, db) .await? .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?; - let module = get_module(&flow_job, module_index); + let module = get_module(&flow_job, &module_step); if module.is_some_and(|x| x.is_flow()) { (false, false, false) } else { (true, se, false) } - } else if is_failure_step { + } else if is_failure_step || matches!(module_step, Step::PreprocessorStep) { (false, false, false) } else { let r = sqlx::query_as::<_, SkipIfStopped>( @@ -346,6 +354,28 @@ pub async fn update_flow_status_after_job_completion_internal< let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into(); + if matches!(module_step, Step::PreprocessorStep) { + sqlx::query!( + "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2", + job_id_for_status, + flow + ).execute(&mut tx).await.map_err(|e| { + Error::InternalErr(format!("error while updating args in preprocessing step: {e:#}")) + })?; + + sqlx::query!( + r#"UPDATE completed_job SET args = '{"reason":"PREPROCESSOR_ARGS_ARE_DISCARDED"}'::jsonb WHERE id = $1"#, + job_id_for_status + ) + .execute(&mut tx) + .await + .map_err(|e| { + Error::InternalErr(format!( + "error while deleting args of preprocessing step: {e:#}" + )) + })?; + } + let (inc_step_counter, new_status) = match module_status { FlowStatusModule::InProgress { iterator, @@ -713,6 +743,21 @@ pub async fn update_flow_status_after_job_completion_internal< "error while setting flow status in failure step: {e:#}" )) })?; + } else if matches!(module_step, Step::PreprocessorStep) { + sqlx::query!( + "UPDATE queue + SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1) + WHERE id = $2", + json!(new_status), + flow + ) + .execute(&mut tx) + .await + .map_err(|e| { + Error::InternalErr(format!( + "error while setting flow status in preprocessing step: {e:#}" + )) + })?; } else { sqlx::query!( "UPDATE queue @@ -824,8 +869,7 @@ pub async fn update_flow_status_after_job_completion_internal< .unwrap_or_else(|| "none".to_string()); tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status"); - let module = get_module(&flow_job, module_index); - + let module = get_module(&flow_job, &module_step); // tracing::error!( // "UPDATE FLOW STATUS 3: {module:#?} {unrecoverable} {} {is_last_step} {success} {skip_error_handler} is_failure_step {is_failure_step}", flow_job.canceled // ); @@ -1115,17 +1159,13 @@ async fn retrieve_flow_jobs_results( Ok(to_raw_value(&results)) } -fn get_module(flow_job: &QueuedJob, module_index: Option) -> Option { +fn get_module(flow_job: &QueuedJob, module_step: &Step) -> Option { let raw_flow = flow_job.parse_raw_flow(); if let Some(raw_flow) = raw_flow { - if let Some(i) = module_index { - if let Some(module) = raw_flow.modules.get(i) { - Some(module.clone()) - } else { - raw_flow.failure_module.map(|x| *x.clone()) - } - } else { - None + match module_step { + Step::PreprocessorStep => raw_flow.preprocessor_module.map(|x| *x.clone()), + Step::Step(i) => raw_flow.modules.get(*i).map(|x| x.clone()), + Step::FailureStep => raw_flow.failure_module.map(|x| *x.clone()), } } else { None @@ -1293,56 +1333,86 @@ pub async fn update_flow_status_in_progress( w_id: &str, flow: Uuid, job_in_progress: Uuid, -) -> error::Result> { +) -> error::Result { let step = get_step_of_flow_status(db, flow).await?; - if let Step::Step(step) = step { - sqlx::query(&format!( - "UPDATE queue - SET flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {step}, job}}', $1), '{{modules, {step}, type}}', $2) - WHERE id = $3 AND workspace_id = $4", - )) - .bind(json!(job_in_progress.to_string())) - .bind(json!("InProgress")) - .bind(flow) - .bind(w_id) - .execute(db) - .await?; - Ok(Some(step)) - } else { - sqlx::query(&format!( - "UPDATE queue - SET flow_status = jsonb_set(jsonb_set(flow_status, '{{failure_module, job}}', $1), '{{failure_module, type}}', $2) - WHERE id = $3 AND workspace_id = $4", - )) - .bind(json!(job_in_progress.to_string())) - .bind(json!("InProgress")) - .bind(flow) - .bind(w_id) - .execute(db) - .await?; - Ok(None) + match step { + Step::Step(step) => { + sqlx::query(&format!( + "UPDATE queue + SET flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {step}, job}}', $1), '{{modules, {step}, type}}', $2) + WHERE id = $3 AND workspace_id = $4", + )) + .bind(json!(job_in_progress.to_string())) + .bind(json!("InProgress")) + .bind(flow) + .bind(w_id) + .execute(db) + .await?; + } + Step::PreprocessorStep => { + sqlx::query(&format!( + "UPDATE queue + SET flow_status = jsonb_set(jsonb_set(flow_status, '{{preprocessor_module, job}}', $1), '{{preprocessor_module, type}}', $2) + WHERE id = $3 AND workspace_id = $4", + )) + .bind(json!(job_in_progress.to_string())) + .bind(json!("InProgress")) + .bind(flow) + .bind(w_id) + .execute(db) + .await?; + } + Step::FailureStep => { + sqlx::query(&format!( + "UPDATE queue + SET flow_status = jsonb_set(jsonb_set(flow_status, '{{failure_module, job}}', $1), '{{failure_module, type}}', $2) + WHERE id = $3 AND workspace_id = $4", + )) + .bind(json!(job_in_progress.to_string())) + .bind(json!("InProgress")) + .bind(flow) + .bind(w_id) + .execute(db) + .await?; + } + } + + Ok(step) +} + +#[derive(Debug)] +pub enum Step { + Step(usize), + PreprocessorStep, + FailureStep, +} + +impl Step { + fn from_i32_and_len(step: i32, len: usize) -> Self { + if step < 0 { + Step::PreprocessorStep + } else if (step as usize) < len { + Step::Step(step as usize) + } else { + Step::FailureStep + } } } -pub enum Step { - Step(i32), - FailureStep, -} #[instrument(level = "trace", skip_all)] pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result { let r = sqlx::query!( - "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1", + "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1", id ) .fetch_one(db) .await .map_err(|e| Error::InternalErr(format!("fetching step flow status: {e:#}")))?; - if r.step < r.len { - Ok(Step::Step(r.step.ok_or_else(|| { - Error::InternalErr("step is null".to_string()) - })?)) + + if let Some(step) = r.step { + Ok(Step::from_i32_and_len(step, r.len.unwrap_or(0) as usize)) } else { - Ok(Step::FailureStep) + Err(Error::InternalErr("step is null".to_string())) } } @@ -1539,14 +1609,20 @@ async fn push_next_flow_job .unwrap_or_else(|| "none".to_string()); tracing::info!(id = %flow_job.id, root_id = %job_root, "pushing next flow job"); - let mut i = usize::try_from(status.step) - .with_context(|| format!("invalid module index {}", status.step))?; + let mut step = Step::from_i32_and_len(status.step, flow.modules.len()); - let mut status_module: FlowStatusModule = status - .modules - .get(i) - .cloned() - .unwrap_or_else(|| status.failure_module.module_status.clone()); + let mut status_module = match step { + Step::Step(i) => status + .modules + .get(i) + .cloned() + .unwrap_or_else(|| status.failure_module.module_status.clone()), + Step::PreprocessorStep => status + .preprocessor_module + .clone() + .unwrap_or_else(|| status.failure_module.module_status.clone()), + Step::FailureStep => status.failure_module.module_status.clone(), + }; let fj: mappable_rc::Marc = flow_job.clone().into(); let arc_flow_job_args: Marc>> = Marc::map(fj, |x| { @@ -1584,7 +1660,7 @@ async fn push_next_flow_job return Ok(()); } - if i == 0 { + if matches!(step, Step::Step(0)) { if !flow_job.is_flow_step && flow_job.schedule_path.is_some() { let no_flow_overlap = sqlx::query_scalar!( "SELECT no_flow_overlap FROM schedule WHERE path = $1 AND workspace_id = $2", @@ -1672,7 +1748,7 @@ async fn push_next_flow_job let arc_last_job_result = if status_module.is_failure() { // if job is being retried, pass the result of its previous failure last_job_result.unwrap_or_else(|| Arc::new(to_raw_value(&json!("{}")))) - } else if i == 0 { + } else if matches!(step, Step::Step(0)) || matches!(step, Step::PreprocessorStep) { // if it's the first job executed in the flow, pass the flow args Arc::new(to_raw_value(&flow_job.args)) } else { @@ -1929,17 +2005,25 @@ async fn push_next_flow_job } } - let mut module: &FlowModule = flow - .modules - .get(i) - .or_else(|| flow.failure_module.as_deref()) - .with_context(|| format!("no module at index {}", status.step))?; + let mut module = match step { + Step::Step(i) => flow + .modules + .get(i) + .with_context(|| format!("no module at index {}", i))?, + Step::PreprocessorStep => flow + .preprocessor_module + .as_ref() + .with_context(|| format!("no preprocessor module"))?, + Step::FailureStep => flow + .failure_module + .as_deref() + .with_context(|| format!("no failure module"))?, + }; let current_id = &module.id; - let mut previous_id = if i >= 1 { - flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap() - } else { - String::new() + let mut previous_id = match step { + Step::Step(i) if i >= 1 => flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap(), + _ => String::new(), }; // calculate sleep if any @@ -1952,10 +2036,13 @@ async fn push_next_flow_job ) { None } else { - let sleep_input_transform = i - .checked_sub(1) - .and_then(|i| flow.modules.get(i)) - .and_then(|m| m.sleep.clone()); + let sleep_input_transform = if let Step::Step(i) = step { + i.checked_sub(1) + .and_then(|i| flow.modules.get(i)) + .and_then(|m| m.sleep.clone()) + } else { + None + }; if let Some(it) = sleep_input_transform { let json_value = match it { @@ -2054,7 +2141,7 @@ async fn push_next_flow_job * The failure module may also run again if it fails and the retry feature is used. * In that case, `i` will index past `flow.modules`. The above should handle that and * re-run the failure module. */ - i = flow.modules.len(); + step = Step::FailureStep; previous_id = current_id.clone(); module = flow .failure_module @@ -2121,6 +2208,13 @@ async fn push_next_flow_job } else { Ok(Marc::new(HashMap::new())) } + } else if matches!(step, Step::PreprocessorStep) { + let mut hm = (*arc_flow_job_args).clone(); + hm.insert( + ENTRYPOINT_OVERRIDE.to_string(), + to_raw_value(&"preprocessor"), + ); + Ok(Marc::new(hm)) } else { match &module.get_value() { Ok( @@ -2522,6 +2616,7 @@ async fn push_next_flow_job tracing::debug!(id = %flow_job.id, root_id = %job_root, "updated parallel monitor lock for {uuid}"); } } + let first_uuid = uuids[0]; let new_status = match next_status { NextStatus::NextLoopIteration { @@ -2610,36 +2705,52 @@ async fn push_next_flow_job } }; - tracing::debug!("STATUS STEP: {:?} {i} {:#?}", status.step, new_status); + tracing::debug!("STATUS STEP: {:?} {step:?} {:#?}", status.step, new_status); - if i >= flow.modules.len() { - sqlx::query!( - "UPDATE queue - SET flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2) - WHERE id = $3", - json!(FlowStatusModuleWParent { - parent_module: Some(current_id.clone()), - module_status: new_status - }), - json!(i), - flow_job.id - ) - .execute(&mut tx) - .await?; - } else { - sqlx::query!( - "UPDATE queue - SET flow_status = JSONB_SET( - JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3) - WHERE id = $4", - i as i32, - json!(new_status), - json!(i), - flow_job.id - ) - .execute(&mut tx) - .await?; + match step { + Step::FailureStep => { + sqlx::query!( + "UPDATE queue + SET flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2) + WHERE id = $3", + json!(FlowStatusModuleWParent { + parent_module: Some(current_id.clone()), + module_status: new_status + }), + json!(flow.modules.len()), + flow_job.id + ) + .execute(&mut tx) + .await?; + } + Step::PreprocessorStep => { + sqlx::query!( + "UPDATE queue + SET flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), ARRAY['step'], $2) + WHERE id = $3", + json!(new_status), + json!(-1), + flow_job.id + ) + .execute(&mut tx) + .await?; + } + Step::Step(i) => { + sqlx::query!( + "UPDATE queue + SET flow_status = JSONB_SET( + JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3) + WHERE id = $4", + i as i32, + json!(new_status), + json!(i), + flow_job.id + ) + .execute(&mut tx) + .await?; + } }; potentially_crash_for_testing(); @@ -2882,9 +2993,17 @@ async fn compute_next_flow_transform( concurrency_time_window_s, .. } => { - let path = path - .clone() - .or_else(|| Some(format!("{}/step-{}", flow_job.script_path(), status.step))); + let path = path.clone().or_else(|| { + if status + .preprocessor_module + .as_ref() + .is_some_and(|x| x.id() == module.id) + { + Some(format!("{}/preprocessor", flow_job.script_path())) + } else { + Some(format!("{}/step-{}", flow_job.script_path(), status.step)) + } + }); let payload = raw_script_to_payload( path, content, @@ -3011,6 +3130,7 @@ async fn compute_next_flow_transform( priority: None, early_return: None, concurrency_key: None, + preprocessor_module: None, }; ContinuePayload::ForloopJobs { n: itered.len(), @@ -3108,6 +3228,7 @@ async fn compute_next_flow_transform( priority: None, early_return: None, concurrency_key: None, + preprocessor_module: None, }, path: Some(format!( "{}/branchone-{}", @@ -3160,6 +3281,7 @@ async fn compute_next_flow_transform( priority: None, early_return: None, concurrency_key: None, + preprocessor_module: None, }, path: Some(format!( "{}/branchall-{}", @@ -3234,6 +3356,7 @@ async fn compute_next_flow_transform( priority: None, early_return: None, concurrency_key: None, + preprocessor_module: None, }, path: Some(format!( "{}/branchall-{}", @@ -3311,6 +3434,7 @@ async fn next_loop_iteration( priority: None, early_return: None, concurrency_key: None, + preprocessor_module: None, }, path: inner_path, restarted_from: None, @@ -3548,7 +3672,11 @@ fn raw_script_to_payload( } fn flow_to_payload(path: &str, delete_after_use: &bool) -> JobPayloadWithTag { - let payload = JobPayload::Flow { path: path.to_string(), dedicated_worker: None }; + let payload = JobPayload::Flow { + path: path.to_string(), + dedicated_worker: None, + apply_preprocessor: false, + }; JobPayloadWithTag { payload, tag: None, delete_after_use: *delete_after_use, timeout: None } } @@ -3562,7 +3690,7 @@ async fn script_to_payload( ) -> Result { let (payload, tag, delete_after_use, script_timeout) = if script_hash.is_none() { let (jp, tag, delete_after_use, script_timeout) = - script_path_to_payload(script_path, db, &flow_job.workspace_id).await?; + script_path_to_payload(script_path, db, &flow_job.workspace_id, Some(true)).await?; ( jp, tag_override.to_owned().or(tag), @@ -3595,6 +3723,7 @@ async fn script_to_payload( language, dedicated_worker, priority, + apply_preprocessor: false, }, tag_override.to_owned().or(tag), delete_after_use, diff --git a/cli/metadata.ts b/cli/metadata.ts index ae145504fe..0753496055 100644 --- a/cli/metadata.ts +++ b/cli/metadata.ts @@ -299,13 +299,15 @@ export async function updateScriptSchema( ): Promise { // infer schema from script content and update it inplace await instantiateWasm(); - const newSchema = inferSchema( + const result = inferSchema( language, scriptContent, metadataContent.schema, path ); - metadataContent.schema = newSchema; + metadataContent.schema = result.schema; + metadataContent.has_preprocessor = result.has_preprocessor; + metadataContent.no_main_func = result.no_main_func; } async function updateScriptLock( @@ -489,7 +491,11 @@ export function inferSchema( `Script ${path} invalid, it cannot be parsed to infer schema.` ) ); - return defaultScriptMetadata().schema; + return { + schema: defaultScriptMetadata().schema, + has_preprocessor: false, + no_main_func: false, + }; } currentSchema.required = []; @@ -515,7 +521,11 @@ export function inferSchema( } } - return currentSchema; + return { + schema: currentSchema, + has_preprocessor: inferedSchema.has_preprocessor, + no_main_func: inferedSchema.no_main_func, + }; } function sortObject(obj: any): any { diff --git a/cli/script.ts b/cli/script.ts index 55b0055e85..6d0b438c7b 100644 --- a/cli/script.ts +++ b/cli/script.ts @@ -97,7 +97,6 @@ async function push(opts: PushOptions, filePath: string) { } export async function findResourceFile(path: string) { - const splitPath = path.split("."); const contentBasePathJSON = splitPath[0] + "." + splitPath[1] + ".json"; @@ -124,9 +123,7 @@ export async function findResourceFile(path: string) { ); } if (validCandidates.length < 1) { - throw new Error( - `No resource matching file resource: ${path}.` - ); + throw new Error(`No resource matching file resource: ${path}.`); } return validCandidates[0]; } @@ -306,6 +303,7 @@ export async function handleFile( restart_unless_cancelled: typed?.restart_unless_cancelled, visible_to_runner_only: typed?.visible_to_runner_only, no_main_func: typed?.no_main_func, + has_preprocessor: typed?.has_preprocessor, priority: typed?.priority, concurrency_key: typed?.concurrency_key, //@ts-ignore @@ -339,6 +337,8 @@ export async function handleFile( Boolean(typed.visible_to_runner_only) == Boolean(remote.visible_to_runner_only) && Boolean(typed.no_main_func) == Boolean(remote.no_main_func) && + Boolean(typed.has_preprocessor) == + Boolean(remote.has_preprocessor) && typed.priority == Boolean(remote.priority) && typed.timeout == remote.timeout && //@ts-ignore diff --git a/cli/wasm/windmill_parser_wasm.generated.d.ts b/cli/wasm/windmill_parser_wasm.generated.d.ts index 10824a1c62..547c600062 100644 --- a/cli/wasm/windmill_parser_wasm.generated.d.ts +++ b/cli/wasm/windmill_parser_wasm.generated.d.ts @@ -19,7 +19,8 @@ export interface InstantiateResult { parse_db_resource: typeof parse_db_resource; parse_graphql: typeof parse_graphql; parse_php: typeof parse_php; - parse_rust: typeof parse_rust + parse_rust: typeof parse_rust; + parse_ansible: typeof parse_ansible }; } @@ -46,9 +47,10 @@ export function instantiateWithInstance(opts?: InstantiateOptions): Promise -// source-hash: d36b5583d8f6223c130a246a8713b8621d55b990 +// source-hash: 6da62ad21f79c5b8fe442f1de939e367943e2665 let wasm; const heap = new Array(128).fill(undefined); @@ -15,6 +15,20 @@ function getObject(idx) { return heap[idx]; } +let heap_next = heap.length; + +function dropObject(idx) { + if (idx < 132) return; + heap[idx] = heap_next; + heap_next = idx; +} + +function takeObject(idx) { + const ret = getObject(idx); + dropObject(idx); + return ret; +} + let WASM_VECTOR_LEN = 0; let cachedUint8Memory0 = null; @@ -89,38 +103,6 @@ function getInt32Memory0() { return cachedInt32Memory0; } -let heap_next = heap.length; - -function dropObject(idx) { - if (idx < 132) return; - heap[idx] = heap_next; - heap_next = idx; -} - -function takeObject(idx) { - const ret = getObject(idx); - dropObject(idx); - return ret; -} - -let cachedFloat64Memory0 = null; - -function getFloat64Memory0() { - if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) { - cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer); - } - return cachedFloat64Memory0; -} - -function addHeapObject(obj) { - if (heap_next === heap.length) heap.push(heap.length + 1); - const idx = heap_next; - heap_next = heap[idx]; - - heap[idx] = obj; - return idx; -} - const cachedTextDecoder = typeof TextDecoder !== "undefined" ? new TextDecoder("utf-8", { ignoreBOM: true, fatal: true }) : { @@ -136,6 +118,24 @@ function getStringFromWasm0(ptr, len) { return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len)); } +function addHeapObject(obj) { + if (heap_next === heap.length) heap.push(heap.length + 1); + const idx = heap_next; + heap_next = heap[idx]; + + heap[idx] = obj; + return idx; +} + +let cachedFloat64Memory0 = null; + +function getFloat64Memory0() { + if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) { + cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer); + } + return cachedFloat64Memory0; +} + let cachedBigInt64Memory0 = null; function getBigInt64Memory0() { @@ -213,11 +213,12 @@ function debugString(val) { } /** * @param {string} code + * @param {string | undefined} [main_override] * @returns {string} */ -export function parse_deno(code) { - let deferred2_0; - let deferred2_1; +export function parse_deno(code, main_override) { + let deferred3_0; + let deferred3_1; try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); const ptr0 = passStringToWasm0( @@ -226,15 +227,23 @@ export function parse_deno(code) { wasm.__wbindgen_realloc, ); const len0 = WASM_VECTOR_LEN; - wasm.parse_deno(retptr, ptr0, len0); + var ptr1 = isLikeNone(main_override) + ? 0 + : passStringToWasm0( + main_override, + wasm.__wbindgen_malloc, + wasm.__wbindgen_realloc, + ); + var len1 = WASM_VECTOR_LEN; + wasm.parse_deno(retptr, ptr0, len0, ptr1, len1); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; - deferred2_0 = r0; - deferred2_1 = r1; + deferred3_0 = r0; + deferred3_1 = r1; return getStringFromWasm0(r0, r1); } finally { wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + wasm.__wbindgen_free(deferred3_0, deferred3_1, 1); } } @@ -375,11 +384,12 @@ export function parse_go(code) { /** * @param {string} code + * @param {string | undefined} [main_override] * @returns {string} */ -export function parse_python(code) { - let deferred2_0; - let deferred2_1; +export function parse_python(code, main_override) { + let deferred3_0; + let deferred3_1; try { const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); const ptr0 = passStringToWasm0( @@ -388,15 +398,23 @@ export function parse_python(code) { wasm.__wbindgen_realloc, ); const len0 = WASM_VECTOR_LEN; - wasm.parse_python(retptr, ptr0, len0); + var ptr1 = isLikeNone(main_override) + ? 0 + : passStringToWasm0( + main_override, + wasm.__wbindgen_malloc, + wasm.__wbindgen_realloc, + ); + var len1 = WASM_VECTOR_LEN; + wasm.parse_python(retptr, ptr0, len0, ptr1, len1); var r0 = getInt32Memory0()[retptr / 4 + 0]; var r1 = getInt32Memory0()[retptr / 4 + 1]; - deferred2_0 = r0; - deferred2_1 = r1; + deferred3_0 = r0; + deferred3_1 = r1; return getStringFromWasm0(r0, r1); } finally { wasm.__wbindgen_add_to_stack_pointer(16); - wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + wasm.__wbindgen_free(deferred3_0, deferred3_1, 1); } } @@ -643,6 +661,33 @@ export function parse_rust(code) { } } +/** + * @param {string} code + * @returns {string} + */ +export function parse_ansible(code) { + let deferred2_0; + let deferred2_1; + try { + const retptr = wasm.__wbindgen_add_to_stack_pointer(-16); + const ptr0 = passStringToWasm0( + code, + wasm.__wbindgen_malloc, + wasm.__wbindgen_realloc, + ); + const len0 = WASM_VECTOR_LEN; + wasm.parse_ansible(retptr, ptr0, len0); + var r0 = getInt32Memory0()[retptr / 4 + 0]; + var r1 = getInt32Memory0()[retptr / 4 + 1]; + deferred2_0 = r0; + deferred2_1 = r1; + return getStringFromWasm0(r0, r1); + } finally { + wasm.__wbindgen_add_to_stack_pointer(16); + wasm.__wbindgen_free(deferred2_0, deferred2_1, 1); + } +} + function handleError(f, args) { try { return f.apply(this, args); @@ -653,6 +698,9 @@ function handleError(f, args) { const imports = { __wbindgen_placeholder__: { + __wbindgen_object_drop_ref: function (arg0) { + takeObject(arg0); + }, __wbindgen_string_get: function (arg0, arg1) { const obj = getObject(arg1); const ret = typeof obj === "string" ? obj : undefined; @@ -667,8 +715,9 @@ const imports = { getInt32Memory0()[arg0 / 4 + 1] = len1; getInt32Memory0()[arg0 / 4 + 0] = ptr1; }, - __wbindgen_object_drop_ref: function (arg0) { - takeObject(arg0); + __wbindgen_error_new: function (arg0, arg1) { + const ret = new Error(getStringFromWasm0(arg0, arg1)); + return addHeapObject(ret); }, __wbindgen_boolean_get: function (arg0) { const v = getObject(arg0); @@ -706,11 +755,7 @@ const imports = { const ret = BigInt.asUintN(64, arg0); return addHeapObject(ret); }, - __wbindgen_error_new: function (arg0, arg1) { - const ret = new Error(getStringFromWasm0(arg0, arg1)); - return addHeapObject(ret); - }, - __wbg_eval_c227cc6614cd76e5: function (arg0, arg1) { + __wbg_eval_b7405fdd08ddef3d: function (arg0, arg1) { const ret = eval(getStringFromWasm0(arg0, arg1)); return addHeapObject(ret); }, @@ -993,6 +1038,7 @@ function getWasmInstanceExports() { parse_graphql, parse_php, parse_rust, + parse_ansible, }; } diff --git a/cli/wasm/windmill_parser_wasm_bg.wasm b/cli/wasm/windmill_parser_wasm_bg.wasm index de163879c8..ef578c3cf2 100644 Binary files a/cli/wasm/windmill_parser_wasm_bg.wasm and b/cli/wasm/windmill_parser_wasm_bg.wasm differ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ba386964a8..712eac83c6 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -66,12 +66,12 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.0.8", "vscode-ws-jsonrpc": "~3.3.2", - "windmill-parser-wasm-go": "^1.382.2", - "windmill-parser-wasm-php": "^1.382.2", - "windmill-parser-wasm-py": "^1.382.2", - "windmill-parser-wasm-regex": "^1.382.2", - "windmill-parser-wasm-rust": "^1.387.0", - "windmill-parser-wasm-ts": "^1.382.2", + "windmill-parser-wasm-go": "^1.397.2", + "windmill-parser-wasm-php": "^1.397.2", + "windmill-parser-wasm-py": "^1.397.2", + "windmill-parser-wasm-regex": "^1.397.2", + "windmill-parser-wasm-rust": "^1.397.2", + "windmill-parser-wasm-ts": "^1.397.2", "windmill-parser-wasm-yaml": "^1.396.1", "windmill-sql-datatype-parser-wasm": "^1.318.0", "y-monaco": "^0.1.4", @@ -13495,34 +13495,34 @@ } }, "node_modules/windmill-parser-wasm-go": { - "version": "1.385.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.385.0.tgz", - "integrity": "sha512-au+i0TDbhnklDrpGTYSXVwqaY4DSKC17anHFLjeG8oHJ9xlc/kNG+2PASs2sqc5m7e6G6D78LDt/5FYs1b9b+g==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-go/-/windmill-parser-wasm-go-1.397.2.tgz", + "integrity": "sha512-je5X7nsbudwlZL6QLpA/s1fwTip0Xmcc8c8eciJjK9IY6JZd2s8IRmtgVb3MB1ppQ4poCzypeqpYcvlL4MOGlg==" }, "node_modules/windmill-parser-wasm-php": { - "version": "1.385.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-php/-/windmill-parser-wasm-php-1.385.0.tgz", - "integrity": "sha512-3x6fFy19BXNfLhzctFEh5fozrKBoWUla/JsZpYaKPw7AKbQ/aP1o+SKZDSrXStaNDHfipdsPYLZ0MIxjZY6p1Q==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-php/-/windmill-parser-wasm-php-1.397.2.tgz", + "integrity": "sha512-+ys3e84UojguDxSJurMT/T2m0XGYUyE59+TbIUteWjxyMnSoynC3GhYvWU3FX0yojwiLOdDaUZkGlkNy8Fl3+w==" }, "node_modules/windmill-parser-wasm-py": { - "version": "1.385.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.385.0.tgz", - "integrity": "sha512-v+6hZNGBp+8KMoAkWIlt5Eu+0n1mNf7vrk/bHr0X2s78cM4JhTXZ24eR7xx69eesQpO7X87pEn/HJHIUHC0kAQ==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-py/-/windmill-parser-wasm-py-1.397.2.tgz", + "integrity": "sha512-gnqQZTRmQi2s3ok9IyixTHGK3MOfJpW2+nnH4W8vXGj1VQMGoXScaZA1W8Nt6/lFy1l+NkAX48mD+nU5s/xbJA==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.385.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.385.0.tgz", - "integrity": "sha512-rgqTANyiYNgdh2Hcgk8xFt/rmLCMGMnIlnZe+/9z2oggfkOKB9oAt49cyCLD9MZeXdTJuNPKn+IVMeSAOn6WYQ==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.397.2.tgz", + "integrity": "sha512-4DhxElRiz9FuwTcMMBhIfk+9cD/oA2+Nf0EBh5BZ1VK4EOD1BVutkTyzWyWP6J+Zqv4m91VyfdVK0Td0qXl+cw==" }, "node_modules/windmill-parser-wasm-rust": { - "version": "1.387.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-rust/-/windmill-parser-wasm-rust-1.387.0.tgz", - "integrity": "sha512-PX0Y/sdaaCkc9rrXB6m+AntGq/RX21fYhdpC7MFCMKb9VakkDAGnFo435xhAi74yLarGd03odUxGymokcBfU6g==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-rust/-/windmill-parser-wasm-rust-1.397.2.tgz", + "integrity": "sha512-7tjekDV2yuqqSjpPF/BfRzaXUwT5eckMWCuq4o18UzChOOeeA4U5goY4INBOGFzYR89m9AeMhwT1IM71nOGrIA==" }, "node_modules/windmill-parser-wasm-ts": { - "version": "1.385.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.385.0.tgz", - "integrity": "sha512-DKQZQzJmSqi7jCbROl9B8ZE/T6sG951zxs1/StWAJmnQazA0FJrdjTj0niiFQNkLaQ2aAPZUkSULZJG+7x1Jow==" + "version": "1.397.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-ts/-/windmill-parser-wasm-ts-1.397.2.tgz", + "integrity": "sha512-nspEC1zLfrvvh6ta8YSjKo99lDfFvsXh7rV2mmpTAmc2ZzzyuVDvW5Cm0Wmo8Ww3seO+oDi/v3jy+8QX8pIzwA==" }, "node_modules/windmill-parser-wasm-yaml": { "version": "1.396.1", diff --git a/frontend/package.json b/frontend/package.json index b0b46ff23b..a66c1c06f8 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -139,12 +139,12 @@ "vscode-languageclient": "~9.0.1", "vscode-uri": "~3.0.8", "vscode-ws-jsonrpc": "~3.3.2", - "windmill-parser-wasm-go": "^1.382.2", - "windmill-parser-wasm-php": "^1.382.2", - "windmill-parser-wasm-py": "^1.382.2", - "windmill-parser-wasm-regex": "^1.382.2", - "windmill-parser-wasm-rust": "^1.387.0", - "windmill-parser-wasm-ts": "^1.382.2", + "windmill-parser-wasm-go": "^1.397.2", + "windmill-parser-wasm-php": "^1.397.2", + "windmill-parser-wasm-py": "^1.397.2", + "windmill-parser-wasm-regex": "^1.397.2", + "windmill-parser-wasm-rust": "^1.397.2", + "windmill-parser-wasm-ts": "^1.397.2", "windmill-parser-wasm-yaml": "^1.396.1", "windmill-sql-datatype-parser-wasm": "^1.318.0", "y-monaco": "^0.1.4", @@ -459,4 +459,4 @@ "optionalDependencies": { "fsevents": "^2.3.3" } -} +} \ No newline at end of file diff --git a/frontend/src/lib/common.ts b/frontend/src/lib/common.ts index ba67c7d651..b96af76d03 100644 --- a/frontend/src/lib/common.ts +++ b/frontend/src/lib/common.ts @@ -143,7 +143,7 @@ export type IntRange = | Exclude, Enumerate> | T -export function pathToMeta(path: string): Meta { +export function pathToMeta(path: string, hideUser: boolean): Meta { const splitted = path.split('/') let ownerKind: OwnerKind if (splitted[0] == 'g') { @@ -155,7 +155,7 @@ export function pathToMeta(path: string): Meta { } else { console.error('Not recognized owner:' + splitted[0]) return { - ownerKind: 'user', + ownerKind: hideUser ? 'folder' : 'user', owner: '', name: '' } diff --git a/frontend/src/lib/components/FlowBuilder.svelte b/frontend/src/lib/components/FlowBuilder.svelte index b6942a7bfd..5de1ecf251 100644 --- a/frontend/src/lib/components/FlowBuilder.svelte +++ b/frontend/src/lib/components/FlowBuilder.svelte @@ -485,6 +485,7 @@ return [ 'settings-metadata', 'constants', + 'preprocessor', ...dfsApply($flowStore.value.modules, (module) => module.id) ] } diff --git a/frontend/src/lib/components/FlowGraphViewer.svelte b/frontend/src/lib/components/FlowGraphViewer.svelte index 22cd5530c2..afeb0438b1 100644 --- a/frontend/src/lib/components/FlowGraphViewer.svelte +++ b/frontend/src/lib/components/FlowGraphViewer.svelte @@ -36,6 +36,7 @@ minHeight={400} modules={flow?.value?.modules} failureModule={flow?.value?.failure_module} + preprocessorModule={flow?.value?.preprocessor_module} on:select={(e) => { const mod = dfs(flow?.value?.modules ?? [], (m) => m).find((m) => m?.id === e?.detail) stepDetail = mod ?? e.detail diff --git a/frontend/src/lib/components/FlowStatusViewerInner.svelte b/frontend/src/lib/components/FlowStatusViewerInner.svelte index 8dd837b2f3..0232548183 100644 --- a/frontend/src/lib/components/FlowStatusViewerInner.svelte +++ b/frontend/src/lib/components/FlowStatusViewerInner.svelte @@ -148,6 +148,9 @@ status?.modules?.concat( status.failure_module.type != 'WaitingForPriorSteps' ? status.failure_module : [] ) ?? [] + if (status.preprocessor_module) { + innerModules.unshift(status.preprocessor_module) + } updateInnerModules() let count = status.retry?.fail_count @@ -789,6 +792,7 @@ {/each} {:else if innerModules.length > 0 && (job.raw_flow?.modules.length ?? 0) > 0} + {@const hasPreprocessor = innerModules[0]?.id == 'preprocessor' ? 1 : 0}

    Step-by-step @@ -798,10 +802,12 @@ {#if render}

    - {#if job?.raw_flow?.modules && i < job?.raw_flow?.modules.length} + {#if mod.id === 'preprocessor'} +

    Preprocessor module

    + {:else if job?.raw_flow?.modules && i < job?.raw_flow?.modules.length + hasPreprocessor} Step - {i + 1} + {i + 1 - hasPreprocessor} out of {job?.raw_flow?.modules.length} @@ -973,6 +979,8 @@ globalRefreshes[detail.moduleId]?.({ job: detail.id, index: detail.index }) }} modules={job.raw_flow?.modules ?? []} + failureModule={job.raw_flow?.failure_module} + preprocessorModule={job.raw_flow?.preprocessor_module} />
    -{#if id && workspace && args && typeof args === 'object' && deepEqual( Object.keys(args), ['reason'] ) && args['reason'] == 'WINDMILL_TOO_BIG'} +{#if args && typeof args === 'object' && deepEqual( Object.keys(args), ['reason'] ) && args['reason'] == 'PREPROCESSOR_ARGS_ARE_DISCARDED'} + Preprocessor args are discarded +{:else if id && workspace && args && typeof args === 'object' && deepEqual( Object.keys(args), ['reason'] ) && args['reason'] == 'WINDMILL_TOO_BIG'} The args are too big in size to be able to fetch alongside job. Please download the JSON file to view them - JobService.runFlowByPath({ workspace: $workspaceStore!, path: val.path, requestBody: args }) + JobService.runFlowByPath({ + workspace: $workspaceStore!, + path: val.path, + requestBody: args, + skipPreprocessor: true + }) ) } else { throw Error('Not supported module type') diff --git a/frontend/src/lib/components/Path.svelte b/frontend/src/lib/components/Path.svelte index 92c9b3a0b7..0b0314d696 100644 --- a/frontend/src/lib/components/Path.svelte +++ b/frontend/src/lib/components/Path.svelte @@ -12,6 +12,7 @@ ResourceService, ScheduleService, ScriptService, + HttpTriggerService, VariableService } from '$lib/gen' import { superadmin, userStore, workspaceStore } from '$lib/stores' @@ -27,7 +28,15 @@ import Tooltip from './Tooltip.svelte' import { Eye, Folder, Plus, SearchCode, User } from 'lucide-svelte' - type PathKind = 'resource' | 'script' | 'variable' | 'flow' | 'schedule' | 'app' | 'raw_app' + type PathKind = + | 'resource' + | 'script' + | 'variable' + | 'flow' + | 'schedule' + | 'app' + | 'raw_app' + | 'http_trigger' let meta: Meta | undefined = undefined export let fullNamePlaceholder: string | undefined = undefined export let namePlaceholder = '' @@ -39,6 +48,7 @@ export let autofocus = true export let dirty = false export let kind: PathKind + export let hideUser: boolean = false let inputP: HTMLInputElement | undefined = undefined @@ -88,19 +98,29 @@ if (path == '' || path == 'u//') { if ($lastMetaUsed == undefined || $lastMetaUsed.owner != $userStore?.username) { meta = { - ownerKind: 'user', + ownerKind: hideUser ? 'folder' : 'user', name: fullNamePlaceholder ?? random_adj() + '_' + namePlaceholder, owner: '' } - if ($userStore?.username?.includes('@')) { - meta.owner = $userStore!.username.split('@')[0].replace(/[^a-zA-Z0-9_]/g, '') - } else { - meta.owner = $userStore!.username! + if (!hideUser) { + if ($userStore?.username?.includes('@')) { + meta.owner = $userStore!.username.split('@')[0].replace(/[^a-zA-Z0-9_]/g, '') + } else { + meta.owner = $userStore!.username! + } } } else { - meta = { - ...$lastMetaUsed, - name: fullNamePlaceholder ?? random_adj() + '_' + namePlaceholder + if ($lastMetaUsed.ownerKind == 'user' && hideUser) { + meta = { + ownerKind: 'folder', + owner: '', + name: fullNamePlaceholder ?? random_adj() + '_' + namePlaceholder + } + } else { + meta = { + ...$lastMetaUsed, + name: fullNamePlaceholder ?? random_adj() + '_' + namePlaceholder + } } } let newMeta = { ...meta } @@ -114,7 +134,7 @@ meta = newMeta path = metaToPath(meta) } else { - meta = pathToMeta(path) + meta = pathToMeta(path, hideUser) } } @@ -195,6 +215,11 @@ return await ScheduleService.existsSchedule({ workspace: $workspaceStore!, path: path }) } else if (kind == 'app') { return await AppService.existsApp({ workspace: $workspaceStore!, path: path }) + } else if (kind == 'http_trigger') { + return await HttpTriggerService.existsHttpTrigger({ + workspace: $workspaceStore!, + path: path + }) } else { return false } @@ -227,14 +252,14 @@ function initPath() { if (path != undefined && path != '') { - meta = pathToMeta(path) + meta = pathToMeta(path, hideUser) onMetaChange() return } if (initialPath == undefined || initialPath == '') { reset() } else { - meta = pathToMeta(initialPath) + meta = pathToMeta(initialPath, hideUser) onMetaChange() path = initialPath } @@ -299,47 +324,49 @@ {#if meta != undefined}
    -
    -   + {#if !hideUser} +
    +   - { - setDirty() - const kind = e.detail - if (meta) { - if (kind === 'folder') { - meta.owner = folders?.[0]?.name ?? '' - } else if (kind === 'group') { - meta.owner = 'all' - } else { - meta.owner = $userStore?.username?.split('@')[0] ?? '' + { + setDirty() + const kind = e.detail + if (meta) { + if (kind === 'folder') { + meta.owner = folders?.[0]?.name ?? '' + } else if (kind === 'group') { + meta.owner = 'all' + } else { + meta.owner = $userStore?.username?.split('@')[0] ?? '' + } } - } - }} - > - - - - -
    + }} + > + + + + +
    + {/if} {#if meta.ownerKind === 'user'}
    - {#if kind != 'app' && kind != 'schedule' && initialPath != '' && initialPath != undefined && initialPath != path} + {#if kind != 'app' && kind != 'schedule' && kind != 'http_trigger' && initialPath != '' && initialPath != undefined && initialPath != path} You are renaming an item that may be depended upon by other items. This may break apps, flows or resources. Find if it used elsewhere using the content search. Note that linked variables diff --git a/frontend/src/lib/components/ResultJobLoader.svelte b/frontend/src/lib/components/ResultJobLoader.svelte index efe00e0940..4510aedff9 100644 --- a/frontend/src/lib/components/ResultJobLoader.svelte +++ b/frontend/src/lib/components/ResultJobLoader.svelte @@ -78,7 +78,8 @@ JobService.runScriptByPath({ workspace: $workspaceStore!, path: path, - requestBody: args + requestBody: args, + skipPreprocessor: true }), callbacks ) @@ -94,7 +95,8 @@ JobService.runScriptByHash({ workspace: $workspaceStore!, hash: hash, - requestBody: args + requestBody: args, + skipPreprocessor: true }), callbacks ) @@ -110,7 +112,8 @@ JobService.runFlowByPath({ workspace: $workspaceStore!, path: path ?? '', - requestBody: args + requestBody: args, + skipPreprocessor: true }), callbacks ) diff --git a/frontend/src/lib/components/ScriptBuilder.svelte b/frontend/src/lib/components/ScriptBuilder.svelte index 8ac175101e..766a733fab 100644 --- a/frontend/src/lib/components/ScriptBuilder.svelte +++ b/frontend/src/lib/components/ScriptBuilder.svelte @@ -231,8 +231,9 @@ } script.schema = script.schema ?? emptySchema() try { - const noMainFunc = await inferArgs(script.language, script.content, script.schema as any) - script.no_main_func = noMainFunc || undefined + const result = await inferArgs(script.language, script.content, script.schema as any) + script.no_main_func = result?.no_main_func || undefined + script.has_preprocessor = result?.has_preprocessor || undefined } catch (error) { sendUserToast(`Could not parse code, are you sure it is valid?`, true) } @@ -263,6 +264,7 @@ concurrency_key: emptyString(script.concurrency_key) ? undefined : script.concurrency_key, visible_to_runner_only: script.visible_to_runner_only, no_main_func: script.no_main_func, + has_preprocessor: script.has_preprocessor, deployment_message: deploymentMsg || undefined } }) @@ -352,8 +354,9 @@ } script.schema = script.schema ?? emptySchema() try { - const noMainFunc = await inferArgs(script.language, script.content, script.schema as any) - script.no_main_func = noMainFunc || undefined + const result = await inferArgs(script.language, script.content, script.schema as any) + script.no_main_func = result?.no_main_func || undefined + script.has_preprocessor = result?.has_preprocessor || undefined } catch (error) { sendUserToast(`Could not parse code, are you sure it is valid?`, true) } @@ -392,7 +395,8 @@ ? undefined : script.concurrency_key, visible_to_runner_only: script.visible_to_runner_only, - no_main_func: script.no_main_func + no_main_func: script.no_main_func, + has_preprocessor: script.has_preprocessor } }) } diff --git a/frontend/src/lib/components/ScriptEditor.svelte b/frontend/src/lib/components/ScriptEditor.svelte index 78064d2453..659585bd1a 100644 --- a/frontend/src/lib/components/ScriptEditor.svelte +++ b/frontend/src/lib/components/ScriptEditor.svelte @@ -25,6 +25,9 @@ import { Clipboard, CornerDownLeft, Github, Play } from 'lucide-svelte' import { setLicense } from '$lib/enterpriseUtils' import type { ScriptEditorWhitelabelCustomUi } from './custom_ui' + import Tabs from './common/tabs/Tabs.svelte' + import Tab from './common/tabs/Tab.svelte' + import { slide } from 'svelte/transition' // Exported export let schema: Schema | any = emptySchema() @@ -105,7 +108,13 @@ // Not defined if JobProgressBar not loaded if (jobProgressReset) jobProgressReset(); //@ts-ignore - testJobLoader.runPreview(path, code, lang, args, tag) + testJobLoader.runPreview( + path, + code, + lang, + selectedTab === 'preprocessor' ? { _ENTRYPOINT_OVERRIDE: 'preprocessor', ...args } : args, + tag + ) } async function loadPastTests(): Promise { @@ -117,11 +126,20 @@ }) } + let hasPreprocessor = false + export async function inferSchema(code: string, nlang?: SupportedLanguage) { let nschema = schema ?? emptySchema() try { - await inferArgs(nlang ?? lang, code, nschema) + const result = await inferArgs( + nlang ?? lang, + code, + nschema, + selectedTab === 'preprocessor' ? 'preprocessor' : undefined + ) + hasPreprocessor = + (selectedTab === 'preprocessor' ? !result?.no_main_func : result?.has_preprocessor) ?? false validCode = true schema = nschema @@ -216,6 +234,11 @@ url.search = '' return `${url}?collab=1` + (edit ? '' : `&path=${path}`) } + let selectedTab: 'main' | 'preprocessor' = 'main' + $: showTabs = hasPreprocessor + $: !hasPreprocessor && (selectedTab = 'main') + + $: selectedTab && inferSchema(code)
    + {#if showTabs} +
    + + Main + Preprocessor + +
    + {/if}
    {#if testIsLoading} + {#if !$copilotInfo.exists_openai_resource_path} +
    +

    + Enable Windmill AI in the + { + close() + }} + > + workspace settings + + +

    +
    + {/if} + +
    + + AI Flow builder + +
    diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte index dbdc85d37e..8e75c6622e 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte @@ -224,8 +224,8 @@ {/if}
    {label}
    -
    - {#if id} +
    + {#if id && id !== 'preprocessor' && !id.startsWith('failure')} {id} {#if deletable} @@ -253,6 +253,7 @@ hover:border-blue-700 hover:!visible {hover ? '' : '!hidden'}" + {#if id !== 'preprocessor'} + {/if} {#if (id && Object.values($flowInputsStore?.[id]?.flowStepWarnings || {}).length > 0) || Boolean(warningMessage)}
    diff --git a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte index 22020da5c9..26244e0336 100644 --- a/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte +++ b/frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte @@ -31,6 +31,8 @@ import { tutorialInProgress } from '$lib/tutorialUtils' import FlowGraphV2 from '$lib/components/graph/FlowGraphV2.svelte' import { replaceId } from '../flowStore' + import { emptySchema } from '$lib/utils' + import { NEVER_TESTED_THIS_FAR } from '../models' export let modules: FlowModule[] | undefined export let sidebarSize: number | undefined = undefined @@ -261,6 +263,7 @@ moving={$moving?.module.id} maxHeight={minHeight} modules={$flowStore.value?.modules} + preprocessorModule={$flowStore.value?.preprocessor_module} {selectedId} {flowInputsStore} on:delete={({ detail }) => { @@ -268,13 +271,15 @@ dependents = getDependentComponents(e.id, $flowStore) const cb = () => { push(history, $flowStore) - - selectNextId(e.id) - - removeAtId($flowStore.value.modules, e.id) - - if ($flowInputsStore) { - delete $flowInputsStore[e.id] + if (e.id === 'preprocessor') { + $selectedId = 'Input' + $flowStore.value.preprocessor_module = undefined + } else { + selectNextId(e.id) + removeAtId($flowStore.value.modules, e.id) + if ($flowInputsStore) { + delete $flowInputsStore[e.id] + } } $flowStore = $flowStore @@ -305,13 +310,26 @@ $selectedId = $moving.module.id $moving = undefined } else { - await insertNewModuleAtIndex( - detail.modules, - detail.index ?? 0, - detail.detail, - detail.script - ) - $selectedId = detail.modules[detail.index ?? 0].id + if (detail.detail === 'preprocessor') { + const preprocessorModule = { + schema: emptySchema(), + previewResult: NEVER_TESTED_THIS_FAR + } + $flowStore.value.preprocessor_module = { + id: 'preprocessor', + value: { type: 'identity' } + } + $flowStateStore['preprocessor'] = preprocessorModule + $selectedId = 'preprocessor' + } else { + await insertNewModuleAtIndex( + detail.modules, + detail.index ?? 0, + detail.detail, + detail.script + ) + $selectedId = detail.modules[detail.index ?? 0].id + } } if (['branchone', 'branchall'].includes(detail.detail)) { diff --git a/frontend/src/lib/components/flows/map/MapItem.svelte b/frontend/src/lib/components/flows/map/MapItem.svelte index c7c78e6a84..a171cd3344 100644 --- a/frontend/src/lib/components/flows/map/MapItem.svelte +++ b/frontend/src/lib/components/flows/map/MapItem.svelte @@ -11,9 +11,11 @@ import { msToSec } from '$lib/utils' import BarsStaggered from '$lib/components/icons/BarsStaggered.svelte' import FlowJobsMenu from './FlowJobsMenu.svelte' + import FlowCopilotButton from './FlowCopilotButton.svelte' export let mod: FlowModule export let insertable: boolean + export let disableAi: boolean export let annotation: string | undefined = undefined export let bgColor: string = '' export let modules: FlowModule[] @@ -60,6 +62,9 @@ {#if mod}
    + {#if insertable && !disableAi && mod.id === 'preprocessor'} + + {/if} {#if moving == mod.id}
    - {#if !$copilotInfo.exists_openai_resource_path} -
    -

    - Enable Windmill AI in the - { - close() - }} - > - workspace settings - - -

    -
    - {/if} - -
    - - AI Flow builder - + + {#if data.insertable && !data.disableAi && !data.hasPreprocessor} + + {/if} + {#if data.insertable && !data.hasPreprocessor} +
    +
    {/if} { data.eventHandlers.delete(e.detail, '') diff --git a/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte b/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte new file mode 100644 index 0000000000..27ab382b51 --- /dev/null +++ b/frontend/src/lib/components/runs/PreprocessedArgsDisplay.svelte @@ -0,0 +1,77 @@ + + +{#if hasPreprocessedArgs} +
    + preprocessed args +
    +{/if} + + diff --git a/frontend/src/lib/components/sidebar/SidebarContent.svelte b/frontend/src/lib/components/sidebar/SidebarContent.svelte index df662f4966..c50b9a8700 100644 --- a/frontend/src/lib/components/sidebar/SidebarContent.svelte +++ b/frontend/src/lib/components/sidebar/SidebarContent.svelte @@ -1,6 +1,6 @@ + +{#if open} + +{/if} diff --git a/frontend/src/lib/components/triggers/RouteEditorInner.svelte b/frontend/src/lib/components/triggers/RouteEditorInner.svelte new file mode 100644 index 0000000000..9e3b62fe27 --- /dev/null +++ b/frontend/src/lib/components/triggers/RouteEditorInner.svelte @@ -0,0 +1,325 @@ + + + + + + {#if !drawerLoading && can_write} + + {/if} + + {#if drawerLoading} + + {:else} +
    +
    + +
    + +
    + {#if !($userStore?.is_admin || $userStore?.is_super_admin)} + + Route endpoints can only be edited by workspace admins + +
    + {/if} +
    + + + + + + + + + +
    +
    + + Full endpoint + + { + currentTarget.select() + }} + /> +
    +
    {routeError}
    +
    +
    +
    + +
    +

    + Pick a script or flow to be triggered
    + To handle headers, query or path parameters, add a preprocessor to your runnable. +

    + +
    +
    +
    +
    +
    Request type
    + + + + +
    +
    +
    Authentication
    + + + + +
    +
    +
    +
    + {/if} +     +     diff --git a/frontend/src/lib/components/triggers/RoutesPanel.svelte b/frontend/src/lib/components/triggers/RoutesPanel.svelte new file mode 100644 index 0000000000..8b14800673 --- /dev/null +++ b/frontend/src/lib/components/triggers/RoutesPanel.svelte @@ -0,0 +1,81 @@ + + + { + loadTriggers() + }} + bind:this={routeEditor} +/> + +
    + {#if $userStore?.is_admin || $userStore?.is_super_admin} + + {/if} +
    + +{#if triggers} + {#if triggers.length == 0} +
    No http routes
    + {:else} +
    + {#each triggers as trigger (trigger.path)} +
    +
    {trigger.path}
    +
    + {trigger.http_method.toUpperCase()} /{trigger.route_path} +
    + +
    + {/each} +
    + {/if} +{:else} + +{/if} diff --git a/frontend/src/lib/infer.ts b/frontend/src/lib/infer.ts index 86df88616a..0bb53957f5 100644 --- a/frontend/src/lib/infer.ts +++ b/frontend/src/lib/infer.ts @@ -32,7 +32,8 @@ import wasmUrlYaml from 'windmill-parser-wasm-yaml/windmill_parser_wasm_bg.wasm? import { workspaceStore } from './stores.js' import { argSigToJsonSchemaType } from './inferArgSig.js' -const loadSchemaLastRun = writable<[string | undefined, MainArgSignature | undefined]>(undefined) +const loadSchemaLastRun = + writable<[string | undefined, MainArgSignature | undefined, string | undefined]>(undefined) let initializeTsPromise: Promise | undefined = undefined export async function initWasmTs() { @@ -63,11 +64,15 @@ async function initWasmYaml() { export async function inferArgs( language: SupportedLanguage | 'bunnative' | undefined, code: string, - schema: Schema -): Promise { + schema: Schema, + mainOverride?: string +): Promise<{ + no_main_func: boolean | null + has_preprocessor: boolean | null +} | null> { const lastRun = get(loadSchemaLastRun) let inferedSchema: MainArgSignature - if (lastRun && code == lastRun[0] && lastRun[1]) { + if (lastRun && code == lastRun[0] && lastRun[1] && lastRun[2] == mainOverride) { inferedSchema = lastRun[1] } else { if (code == '') { @@ -81,16 +86,16 @@ export async function inferArgs( } if (language == 'python3') { await initWasmPython() - inferedSchema = JSON.parse(parse_python(code)) + inferedSchema = JSON.parse(parse_python(code, mainOverride)) } else if (language == 'deno') { await initWasmTs() - inferedSchema = JSON.parse(parse_deno(code)) + inferedSchema = JSON.parse(parse_deno(code, mainOverride)) } else if (language == 'nativets') { await initWasmTs() - inferedSchema = JSON.parse(parse_deno(code)) + inferedSchema = JSON.parse(parse_deno(code, mainOverride)) } else if (language == 'bun' || language == 'bunnative') { await initWasmTs() - inferedSchema = JSON.parse(parse_deno(code)) + inferedSchema = JSON.parse(parse_deno(code, mainOverride)) } else if (language == 'postgresql') { inferedSchema = JSON.parse(parse_sql(code)) if (inlineDBResource === undefined) { @@ -162,7 +167,7 @@ export async function inferArgs( if (inferedSchema.type == 'Invalid') { throw new Error(inferedSchema.error) } - loadSchemaLastRun.set([code, inferedSchema]) + loadSchemaLastRun.set([code, inferedSchema, mainOverride]) } schema.required = [] @@ -186,7 +191,10 @@ export async function inferArgs( } await tick() - return inferedSchema.no_main_func + return { + no_main_func: inferedSchema.no_main_func, + has_preprocessor: inferedSchema.has_preprocessor + } } export async function loadSchemaFromPath(path: string, hash?: string): Promise { diff --git a/frontend/src/lib/script_helpers.ts b/frontend/src/lib/script_helpers.ts index 2cc38b6a54..458499d3d2 100644 --- a/frontend/src/lib/script_helpers.ts +++ b/frontend/src/lib/script_helpers.ts @@ -287,8 +287,6 @@ fn main(who_to_greet: String, numbers: Vec) -> anyhow::Result { } ` - - export const FETCH_INIT_CODE = `export async function main( url: string | undefined, method: 'GET' | 'POST' | 'PUT' | 'DELETE' | 'PATCH' | 'OPTIONS' = 'GET', @@ -431,6 +429,48 @@ export async function main(approver?: string) { // add a form in Advanced - Suspend // all on approval steps: https://www.windmill.dev/docs/flows/flow_approval` +export const BUN_PREPROCESSOR_MODULE_CODE = ` +export async function preprocessor( + wm_trigger: { + kind: 'http' | 'email' | 'webhook', + http?: { + route: string // The route path, e.g. "/users/:id" + path: string // The actual path called, e.g. "/users/123" + method: string + params: Record + query: Record + headers: Record + } + }, + /* your other args */ +) { + return { + // return the args to be passed to the flow + } +} +` + +export const DENO_PREPROCESSOR_MODULE_CODE = ` +export async function preprocessor( + wm_trigger: { + kind: 'http' | 'email' | 'wehbook', + http?: { + route: string // The route path, e.g. "/users/:id" + path: string // The actual path called, e.g. "/users/123" + method: string + params: Record + query: Record + headers: Record + } + }, + /* your other args */ +) { + return { + // return the args to be passed to the flow + } +} +` + export const PYTHON_INIT_CODE_APPROVAL = `import wmill def main(): @@ -458,6 +498,29 @@ def main(): # add a form in Advanced - Suspend # all on approval steps: https://www.windmill.dev/docs/flows/flow_approval` +export const PYTHON_PREPROCESSOR_MODULE_CODE = `from typing import TypedDict, Literal + +class Http(TypedDict): + route: str # The route path, e.g. "/users/:id" + path: str # The actual path called, e.g. "/users/123" + method: str + params: dict[str, str] + query: dict[str, str] + headers: dict[str, str] + +class WmTrigger(TypedDict): + kind: Literal["http", "email", "webhook"] + http: Http | None + +def preprocessor( + wm_trigger: WmTrigger, + # your other args +): + return { + # return the args to be passed to the flow + } +` + export const DOCKER_INIT_CODE = `# shellcheck shell=bash # Bash script that calls docker as a client to the host daemon # See documentation: https://www.windmill.dev/docs/advanced/docker @@ -540,16 +603,21 @@ const ALL_INITIAL_CODE = [ DENO_INIT_CODE_TRIGGER, DENO_INIT_CODE_CLEAR, PYTHON_INIT_CODE_CLEAR, + PYTHON_FAILURE_MODULE_CODE, + PYTHON_PREPROCESSOR_MODULE_CODE, DENO_INIT_CODE_APPROVAL, DENO_FAILURE_MODULE_CODE, + DENO_PREPROCESSOR_MODULE_CODE, BUN_INIT_CODE, BUN_INIT_CODE_CLEAR, BUN_INIT_CODE_APPROVAL, + BUN_FAILURE_MODULE_CODE, + BUN_PREPROCESSOR_MODULE_CODE, BASH_INIT_CODE, POWERSHELL_INIT_CODE, PHP_INIT_CODE, RUST_INIT_CODE, - ANSIBLE_PLAYBOOK_INIT_CODE, + ANSIBLE_PLAYBOOK_INIT_CODE ] export function isInitialCode(content: string): boolean { @@ -573,6 +641,7 @@ export function initialCode( | 'docker' | 'powershell' | 'bunnative' + | 'preprocessor' | undefined ): string { if (!kind) { @@ -590,6 +659,8 @@ export function initialCode( return MYSQL_INIT_CODE } else if (subkind === 'fetch') { return FETCH_INIT_CODE + } else if (subkind === 'preprocessor') { + return DENO_PREPROCESSOR_MODULE_CODE } else { return DENO_INIT_CODE } @@ -609,6 +680,8 @@ export function initialCode( return PYTHON_FAILURE_MODULE_CODE } else if (subkind === 'flow') { return PYTHON_INIT_CODE_CLEAR + } else if (subkind === 'preprocessor') { + return PYTHON_PREPROCESSOR_MODULE_CODE } else { return PYTHON_INIT_CODE } @@ -647,6 +720,8 @@ export function initialCode( return BUN_INIT_CODE_APPROVAL } else if (kind === 'failure') { return BUN_FAILURE_MODULE_CODE + } else if (subkind === 'preprocessor') { + return BUN_PREPROCESSOR_MODULE_CODE } if (subkind === 'flow') { return BUN_INIT_CODE_CLEAR diff --git a/frontend/src/lib/stores.ts b/frontend/src/lib/stores.ts index aa65ceb8bc..18cf32b074 100644 --- a/frontend/src/lib/stores.ts +++ b/frontend/src/lib/stores.ts @@ -89,6 +89,7 @@ export const vimMode = writable(getLocalSetting(VIM_MODE_SETTING_NAME) export const codeCompletionSessionEnabled = writable( getLocalSetting(CODE_COMPLETION_SETTING_NAME) != 'false' ) +export const usedTriggerKinds = writable([]) type SQLBaseSchema = { [schemaKey: string]: { diff --git a/frontend/src/routes/(root)/(logged)/+layout.svelte b/frontend/src/routes/(root)/(logged)/+layout.svelte index 9b9f6ec35c..3eae42916f 100644 --- a/frontend/src/routes/(root)/(logged)/+layout.svelte +++ b/frontend/src/routes/(root)/(logged)/+layout.svelte @@ -8,6 +8,7 @@ RawAppService, ScriptService, SettingService, + HttpTriggerService, UserService, WorkspaceService } from '$lib/gen' @@ -25,7 +26,8 @@ workspaceStore, type UserExt, defaultScripts, - hubBaseUrlStore + hubBaseUrlStore, + usedTriggerKinds } from '$lib/stores' import CenteredModal from '$lib/components/CenteredModal.svelte' import { afterNavigate, beforeNavigate } from '$app/navigation' @@ -124,6 +126,7 @@ loadUsage() syncTutorialsTodos() loadHubBaseUrl() + loadUsedTriggerKinds() } async function loadUsage() { @@ -183,6 +186,11 @@ ] } + async function loadUsedTriggerKinds() { + const httpUsed = await HttpTriggerService.used({ workspace: $workspaceStore ?? '' }) + $usedTriggerKinds = httpUsed ? ['http'] : [] + } + function pathInAppMode(pathname: string | undefined): boolean { if (!pathname) return false return ( diff --git a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte index 5832e9e444..3f71923a15 100644 --- a/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/flows/get/[...path]/+page.svelte @@ -55,6 +55,7 @@ import FlowHistory from '$lib/components/flows/FlowHistory.svelte' import EmailTriggerPanel from '$lib/components/details/EmailTriggerPanel.svelte' import Star from '$lib/components/Star.svelte' + import RoutesPanel from '$lib/components/triggers/RoutesPanel.svelte' let flow: Flow | undefined let can_write = false @@ -130,7 +131,8 @@ invisibleToOwner, requestBody: args, scheduledFor, - tag: overrideTag + tag: overrideTag, + skipPreprocessor: true }) await goto('/run/' + run + '?workspace=' + $workspaceStore) } catch (e) { @@ -492,6 +494,9 @@ {args} /> + + + + import { HttpTriggerService, type HttpTrigger } from '$lib/gen' + import { canWrite, displayDate, getLocalSetting, storeLocalSetting } from '$lib/utils' + import { base } from '$app/paths' + import CenteredPage from '$lib/components/CenteredPage.svelte' + import { Button, Skeleton } from '$lib/components/common' + import Dropdown from '$lib/components/DropdownV2.svelte' + import PageHeader from '$lib/components/PageHeader.svelte' + import RouteEditor from '$lib/components/triggers/RouteEditor.svelte' + import SharedBadge from '$lib/components/SharedBadge.svelte' + import ShareModal from '$lib/components/ShareModal.svelte' + import Toggle from '$lib/components/Toggle.svelte' + import { userStore, workspaceStore } from '$lib/stores' + import { Route, Code, Eye, Pen, Plus, Share, Trash } from 'lucide-svelte' + import { goto } from '$lib/navigation' + import SearchItems from '$lib/components/SearchItems.svelte' + import NoItemFound from '$lib/components/home/NoItemFound.svelte' + import RowIcon from '$lib/components/common/table/RowIcon.svelte' + import ListFilters from '$lib/components/home/ListFilters.svelte' + import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte' + import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte' + import { setQuery } from '$lib/navigation' + import { onMount } from 'svelte' + + type TriggerW = HttpTrigger & { canWrite: boolean } + + let triggers: TriggerW[] = [] + let shareModal: ShareModal + let loading = true + + async function loadTriggers(): Promise { + triggers = (await HttpTriggerService.listHttpTriggers({ workspace: $workspaceStore! })).map( + (x) => { + return { canWrite: canWrite(x.path, x.extra_perms!, $userStore), ...x } + } + ) + loading = false + } + + $: { + if ($workspaceStore && $userStore) { + loadTriggers() + } + } + let routeEditor: RouteEditor + + let filteredItems: (TriggerW & { marked?: any })[] | undefined = [] + let items: typeof filteredItems | undefined = [] + let preFilteredItems: typeof filteredItems | undefined = [] + let filter = '' + let ownerFilter: string | undefined = undefined + let nbDisplayed = 15 + + const TRIGGER_PATH_KIND_FILTER_SETTING = 'filter_path_of' + const FILTER_USER_FOLDER_SETTING_NAME = 'user_and_folders_only' + let selectedFilterKind = + (getLocalSetting(TRIGGER_PATH_KIND_FILTER_SETTING) as 'route' | 'script_flow') ?? 'route' + let filterUserFolders = getLocalSetting(FILTER_USER_FOLDER_SETTING_NAME) == 'true' + + $: storeLocalSetting(TRIGGER_PATH_KIND_FILTER_SETTING, selectedFilterKind) + $: storeLocalSetting(FILTER_USER_FOLDER_SETTING_NAME, filterUserFolders ? 'true' : undefined) + + function filterItemsPathsBaseOnUserFilters( + item: TriggerW, + selectedFilterKind: 'route' | 'script_flow', + filterUserFolders: boolean + ) { + if ($workspaceStore == 'admins') return true + if (filterUserFolders) { + if (selectedFilterKind === 'route') { + return ( + !item.path.startsWith('u/') || item.path.startsWith('u/' + $userStore?.username + '/') + ) + } else { + return ( + !item.script_path.startsWith('u/') || + item.script_path.startsWith('u/' + $userStore?.username + '/') + ) + } + } else { + return true + } + } + + $: preFilteredItems = + ownerFilter != undefined + ? selectedFilterKind === 'route' + ? triggers?.filter( + (x) => + x.path.startsWith(ownerFilter + '/' ?? '') && + filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) + ) + : triggers?.filter( + (x) => + x.script_path.startsWith(ownerFilter + '/' ?? '') && + filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) + ) + : triggers?.filter((x) => + filterItemsPathsBaseOnUserFilters(x, selectedFilterKind, filterUserFolders) + ) + + $: if ($workspaceStore) { + ownerFilter = undefined + } + + $: owners = + selectedFilterKind === 'route' + ? Array.from( + new Set(filteredItems?.map((x) => x.path.split('/').slice(0, 2).join('/')) ?? []) + ).sort() + : Array.from( + new Set(filteredItems?.map((x) => x.script_path.split('/').slice(0, 2).join('/')) ?? []) + ).sort() + + $: items = filter !== '' ? filteredItems : preFilteredItems + + function updateQueryFilters(selectedFilterKind, filterUserFolders) { + setQuery( + new URL(window.location.href), + TRIGGER_PATH_KIND_FILTER_SETTING, + selectedFilterKind + ).then(() => { + setQuery( + new URL(window.location.href), + FILTER_USER_FOLDER_SETTING_NAME, + String(filterUserFolders) + ) + }) + } + + function loadQueryFilters() { + let url = new URL(window.location.href) + let queryFilterKind = url.searchParams.get(TRIGGER_PATH_KIND_FILTER_SETTING) + let queryFilterUserFolders = url.searchParams.get(FILTER_USER_FOLDER_SETTING_NAME) + if (queryFilterKind) { + selectedFilterKind = queryFilterKind as 'route' | 'script_flow' + } + if (queryFilterUserFolders) { + filterUserFolders = queryFilterUserFolders == 'true' + } + } + + onMount(() => { + loadQueryFilters() + }) + + $: updateQueryFilters(selectedFilterKind, filterUserFolders) + + + + + (x.summary ?? '') + ' ' + x.path + ' (' + x.script_path + ')'} +/> + + + + {#if $userStore?.is_admin || $userStore?.is_super_admin} + + {/if} + +
    +
    + +
    +
    Filter by path of
    + + + + +
    + + +
    + {#if $userStore?.is_super_admin && $userStore.username.includes('@')} + + {:else if $userStore?.is_admin || $userStore?.is_super_admin} + + {/if} +
    +
    + {#if loading} + {#each new Array(6) as _} + + {/each} + {:else if !triggers?.length} +
    No routes
    + {:else if items?.length} +
    + {#each items.slice(0, nbDisplayed) as { path, edited_by, edited_at, script_path, route_path, is_flow, extra_perms, canWrite, marked, http_method } (path)} + {@const href = `${is_flow ? '/flows/get' : '/scripts/get'}/${script_path}`} + +
    +
    + + + routeEditor?.openEdit(path, is_flow)} + class="min-w-0 grow hover:underline decoration-gray-400" + > +
    + {#if marked} + + {@html marked} + + {:else} + {http_method.toUpperCase()} /{route_path} + {/if} +
    +
    + {path} +
    +
    + runnable: {script_path} +
    +     + + + +
    + + { + goto(href) + } + }, + { + displayName: 'Delete', + type: 'delete', + icon: Trash, + disabled: !canWrite || !($userStore?.is_admin || $userStore?.is_super_admin), + action: async () => { + await HttpTriggerService.deleteHttpTrigger({ + workspace: $workspaceStore ?? '', + path + }) + loadTriggers() + } + }, + { + displayName: canWrite ? 'Edit' : 'View', + icon: canWrite ? Pen : Eye, + action: () => { + routeEditor?.openEdit(path, is_flow) + } + }, + { + displayName: 'Audit logs', + icon: Eye, + href: `${base}/audit_logs?resource=${path}` + }, + { + displayName: canWrite ? 'Share' : 'See Permissions', + icon: Share, + action: () => { + shareModal.openDrawer(path, 'http_trigger') + } + } + ]} + /> +
    +
    +
    +
    edited by {edited_by}
    the {displayDate(edited_at)}
    +
    + {/each} +
    + {:else} + + {/if} +
    + {#if items && items?.length > 15 && nbDisplayed < items.length} + {nbDisplayed} items out of {items.length} + + {/if} +     + + { + loadTriggers() + }} +/> diff --git a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte index 4ae0562272..8ab81a1e38 100644 --- a/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte +++ b/frontend/src/routes/(root)/(logged)/run/[...run]/+page.svelte @@ -91,6 +91,7 @@ import ScheduleEditor from '$lib/components/ScheduleEditor.svelte' import Popover from '$lib/components/Popover.svelte' import HighlightTheme from '$lib/components/HighlightTheme.svelte' + import PreprocessedArgsDisplay from '$lib/components/runs/PreprocessedArgsDisplay.svelte' let job: Job | undefined let jobUpdateLastFetch: Date | undefined @@ -608,7 +609,11 @@
    {/if} + {#if job && job.flow_status && job.job_kind === 'script'} + + {/if} {#if persistentScriptDefinition} +
    diff --git a/frontend/src/routes/view_graph/+page.svelte b/frontend/src/routes/view_graph/+page.svelte index 18fbac237e..6844733561 100644 --- a/frontend/src/routes/view_graph/+page.svelte +++ b/frontend/src/routes/view_graph/+page.svelte @@ -3,15 +3,20 @@ import { decodeState } from '$lib/utils' let content = localStorage.getItem('svelvet') - const { modules, failureModule } = content + const { modules, failureModule, preprocessorModule } = content ? decodeState(content) - : { modules: [], failureModule: undefined } + : { modules: [], failureModule: undefined, preprocessorModule: undefined } - + DownloadDownload diff --git a/openflow.openapi.yaml b/openflow.openapi.yaml index a0bec8e72b..9d5add0348 100644 --- a/openflow.openapi.yaml +++ b/openflow.openapi.yaml @@ -44,6 +44,8 @@ components: $ref: "#/components/schemas/FlowModule" failure_module: $ref: "#/components/schemas/FlowModule" + preprocessor_module: + $ref: "#/components/schemas/FlowModule" same_worker: type: boolean concurrent_limit: @@ -439,6 +441,9 @@ components: $ref: "#/components/schemas/FlowStatusModule" user_states: additionalProperties: true + preprocessor_module: + allOf: + - $ref: "#/components/schemas/FlowStatusModule" failure_module: allOf: - $ref: "#/components/schemas/FlowStatusModule"