starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-28 00:02:42 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(frontend): add support for routes

Browse Source
This commit is contained in:
Faton Ramadani
2024-09-24 15:19:50 +02:00
parent 519566b7b8 ad69c08228
commit 65120393c5
172 changed files with 5580 additions and 1208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/docker-image.yml
+2 -4
View File
@@ -1,10 +1,8 @@
env:
REGISTRY: ghcr.io
IMAGE_NAME:
${{ github.event_name != 'pull_request' && github.repository ||
IMAGE_NAME: ${{ github.event_name != 'pull_request' && github.repository ||
'windmill-labs/windmill-test' }}
DEV_SHA:
${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}',
DEV_SHA: ${{ github.event_name != 'pull_request' && 'dev' || format('pr-{0}',
github.event.number) }}
name: Build windmill:main
CHANGELOG.md
+46
View File
@@ -1,5 +1,51 @@
# Changelog
## [1.398.1](https://github.com/windmill-labs/windmill/compare/v1.398.0...v1.398.1) (2024-09-23)
### Bug Fixes
* time handling in bun and deno ([#4423](https://github.com/windmill-labs/windmill/issues/4423)) ([61f27ac](https://github.com/windmill-labs/windmill/commit/61f27acbd5b477f0e2453d4b02d406a7aef76009))
## [1.398.0](https://github.com/windmill-labs/windmill/compare/v1.397.4...v1.398.0) (2024-09-23)
### Features
* **frontend:** add http routing templates ([#4421](https://github.com/windmill-labs/windmill/issues/4421)) ([e99e7b2](https://github.com/windmill-labs/windmill/commit/e99e7b2b0bb1bc63d1253d147b80bac06eaff103))
* http routing ([#4339](https://github.com/windmill-labs/windmill/issues/4339)) ([304dac3](https://github.com/windmill-labs/windmill/commit/304dac34475b8871621c54a768275fae1e9f845e))
### Bug Fixes
* allow no body in job requests ([#4413](https://github.com/windmill-labs/windmill/issues/4413)) ([70fa78c](https://github.com/windmill-labs/windmill/commit/70fa78c10d8afb3e06be84e3449fdfdeb2673765))
* **frontend:** Fix delete branch one index ([#4418](https://github.com/windmill-labs/windmill/issues/4418)) ([30017cc](https://github.com/windmill-labs/windmill/commit/30017cc2b1401ea439e3fbec5d7764b206100a38))
* migrate smtp instance settings to global settings ([#4416](https://github.com/windmill-labs/windmill/issues/4416)) ([50a6f78](https://github.com/windmill-labs/windmill/commit/50a6f789fa8a03d70407b0429da049cc0740b6fa))
* no failed renewal alert if trial ([#4414](https://github.com/windmill-labs/windmill/issues/4414)) ([5be7be0](https://github.com/windmill-labs/windmill/commit/5be7be03a6df34a9fae96ed7f844d693f7cfce3e))
* update pip to 24.2 and python 3.11.8->3.11.10 ([a17195d](https://github.com/windmill-labs/windmill/commit/a17195d88ff52b72f0c2cdd95c6218df8ef6d7cf))
* various improvements for ansible ([#4419](https://github.com/windmill-labs/windmill/issues/4419)) ([a500994](https://github.com/windmill-labs/windmill/commit/a500994cc2cd9cc6cbad722059cebd0bf74a8850))
## [1.397.4](https://github.com/windmill-labs/windmill/compare/v1.397.3...v1.397.4) (2024-09-20)
### Bug Fixes
* **cli:** make CLI not require latest version of windmill ([794f87a](https://github.com/windmill-labs/windmill/commit/794f87aa0d33a0b782fcb2726754fdc51bc1d73d))
## [1.397.3](https://github.com/windmill-labs/windmill/compare/v1.397.2...v1.397.3) (2024-09-20)
### Bug Fixes
* **cli:** make CLI compatible with Node 18 ([8212532](https://github.com/windmill-labs/windmill/commit/8212532b29fbcf971cb320b8cfc84f3a8d8ad795))
## [1.397.2](https://github.com/windmill-labs/windmill/compare/v1.397.1...v1.397.2) (2024-09-20)
### Bug Fixes
* **bun:** never pre-bundle BASE_URL env variable ([69b8754](https://github.com/windmill-labs/windmill/commit/69b8754aef32682bf812be783f79b81cd8526c7a))
## [1.397.1](https://github.com/windmill-labs/windmill/compare/v1.397.0...v1.397.1) (2024-09-20)
Dockerfile
+3 -1
View File
@@ -1,6 +1,6 @@
ARG DEBIAN_IMAGE=debian:bookworm-slim
ARG RUST_IMAGE=rust:1.80-slim-bookworm
ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm
ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm
FROM ${RUST_IMAGE} AS rust_base
@@ -95,6 +95,8 @@ ARG WITH_KUBECTL=true
ARG WITH_HELM=true
ARG WITH_GIT=true
RUN pip install --upgrade pip==24.2
RUN apt-get update \
&& apt-get install -y ca-certificates wget curl jq unzip build-essential unixodbc xmlsec1 software-properties-common \
&& apt-get clean \
backend/.sqlx/query-02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441.json
Generated
+89
View File
@@ -0,0 +1,89 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "route_path",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "is_async",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "requires_auth",
"type_info": "Bool"
},
{
"ordinal": 7,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "http_method: _",
"type_info": {
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
}
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false,
false
]
},
"hash": "02f1a6eeb27067dc438459238e7b016f5ccf9e3fe0ffbe88471f15aad8f74441"
}
backend/.sqlx/query-0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90.json
Generated
-16
View File
@@ -1,16 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n WHERE id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Jsonb",
"Uuid"
]
},
"nullable": []
},
"hash": "0aff8b0f2cddc0dd696759f48368c7315ff39060be5430c59582e1e05a4b3d90"
}
backend/.sqlx/query-16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63.json
Generated
+36
View File
@@ -0,0 +1,36 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
}
]
},
"nullable": [
null
]
},
"hash": "16c4fb122ceb30722f54c92db97427b0ddd36f495d549f8cd23d659d2c866a63"
}
backend/.sqlx/query-17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), ARRAY['step'], $2)\n WHERE id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Jsonb",
"Uuid"
]
},
"nullable": []
},
"hash": "17e3e2a3232865c33fa535b5d99455942e30f932c5b97f1e5b508128f39a288f"
}
backend/.sqlx/query-872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e.json → backend/.sqlx/query-3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d.json
Generated
+4 -4
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return \n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2",
"query": "SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor \n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2",
"describe": {
"columns": [
{
@@ -15,8 +15,8 @@
},
{
"ordinal": 2,
"name": "early_return",
"type_info": "Text"
"name": "has_preprocessor",
"type_info": "Bool"
}
],
"parameters": {
@@ -31,5 +31,5 @@
null
]
},
"hash": "872dcaec230579e4480adf23075e323557efbe52c813e3a6a0da6b855291951e"
"hash": "3fedf12ee5d44e8af9c3864cc70a8f607dccc015ce2d5150c56c68074cd0f84d"
}
backend/.sqlx/query-4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE completed_job SET args = '{\"reason\":\"PREPROCESSOR_ARGS_ARE_DISCARDED\"}'::jsonb WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "4c97fcc93b31c4b3262419d6ee183773a95de3b6694a398cb80d288bef4f130f"
}
backend/.sqlx/query-4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea.json
Generated
+38
View File
@@ -0,0 +1,38 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE http_trigger \n SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now() \n WHERE workspace_id = $11 AND path = $12",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Bool",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
},
"Varchar",
"Varchar",
"Bool",
"Bool",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "4d8640e84fccf1a0b799d8396a51e69345137e68d5096c70ba0a4332075d97ea"
}
backend/.sqlx/query-4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491.json
Generated
+108
View File
@@ -0,0 +1,108 @@
{
"db_name": "PostgreSQL",
"query": "SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as \"http_method: _\", edited_by, email, edited_at, extra_perms, is_async, requires_auth\n FROM http_trigger\n WHERE workspace_id = $1 AND path = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "route_path",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "route_path_key",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "http_method: _",
"type_info": {
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
}
},
{
"ordinal": 7,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "edited_at",
"type_info": "Timestamptz"
},
{
"ordinal": 10,
"name": "extra_perms",
"type_info": "Jsonb"
},
{
"ordinal": 11,
"name": "is_async",
"type_info": "Bool"
},
{
"ordinal": 12,
"name": "requires_auth",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
false,
false
]
},
"hash": "4fb95eae1c871241efe2ef79615ce03cba0e4a12aad3274e4829d98e38ca1491"
}
backend/.sqlx/query-55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0.json
Generated
+36
View File
@@ -0,0 +1,36 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now() \n WHERE workspace_id = $9 AND path = $10",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bool",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
},
"Varchar",
"Varchar",
"Bool",
"Bool",
"Text",
"Text"
]
},
"nullable": []
},
"hash": "55d44f569f8ebfccddf975e1a330ef0dc286f4138efe923832371cdbac7157b0"
}
backend/.sqlx/query-8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c.json → backend/.sqlx/query-64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2",
"query": "select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where hash = $1 AND workspace_id = $2",
"describe": {
"columns": [
{
@@ -82,6 +82,11 @@
"ordinal": 10,
"name": "timeout",
"type_info": "Int4"
},
{
"ordinal": 11,
"name": "has_preprocessor",
"type_info": "Bool"
}
],
"parameters": {
@@ -101,8 +106,9 @@
true,
true,
true,
true,
true
]
},
"hash": "8b10c9ade85c0307e300224e87962ceba8e7ccbe53b56c4670a1634bb9d0e89c"
"hash": "64ce3eabed6de7a4f99884640c21218ac19ab2a4bc5f8b42deb2476f85d0deac"
}
backend/.sqlx/query-661f472ff3860983322162420457f5033b9c9afc344d9c3e385ba20a3ad2197a.json
Generated
+1 -1
View File
@@ -5,7 +5,7 @@
"columns": [
{
"ordinal": 0,
"name": "bool",
"name": "?column?",
"type_info": "Bool"
}
],
backend/.sqlx/query-8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3.json → backend/.sqlx/query-83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b.json
Generated
+4 -4
View File
@@ -1,11 +1,11 @@
{
"db_name": "PostgreSQL",
"query": "SELECT config FROM config WHERE name = 'server'",
"query": "SELECT value FROM global_settings WHERE name = 'smtp_settings'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "config",
"name": "value",
"type_info": "Jsonb"
}
],
@@ -13,8 +13,8 @@
"Left": []
},
"nullable": [
true
false
]
},
"hash": "8ca76fcadc659d7069b97930cdde2d5200f3cef9dedb83b76c8a6a433d2a4de3"
"hash": "83ec97f6aad154e0e06ee05a3647dab8f89b1b2d7a569c7eac8c4169e37b9f8b"
}
backend/.sqlx/query-867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd.json
Generated
+37
View File
@@ -0,0 +1,37 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
}
]
},
"nullable": [
null
]
},
"hash": "867e0f8299e22a6b8e02d21adeb0281b1a10593715db823a5adeeac3ab1058cd"
}
backend/.sqlx/query-870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368.json
Generated
+17
View File
@@ -0,0 +1,17 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n WHERE id = $4",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Jsonb",
"Jsonb",
"Uuid"
]
},
"nullable": []
},
"hash": "870d7feec169f70d140b7477562315ed2fd4662975bf0d01cfbffac86a959368"
}
backend/.sqlx/query-98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba.json
Generated
+41
View File
@@ -0,0 +1,41 @@
{
"db_name": "PostgreSQL",
"query": "SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor\n FROM flow \n LEFT JOIN flow_version\n ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]\n WHERE flow.path = $1 and flow.workspace_id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "tag",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "dedicated_worker",
"type_info": "Bool"
},
{
"ordinal": 2,
"name": "early_return",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "has_preprocessor",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
true,
true,
null,
null
]
},
"hash": "98081ee9a4a4a52ae70d74d26e8608a010722d7ea9dc343ae8f03e38978ffaba"
}
backend/.sqlx/query-9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE path = $1 AND workspace_id = $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "9a050ab74cfc13a4b855408e61ebcb0a9d27e5563fa7af5e00a4916b03a62ddb"
}
backend/.sqlx/query-9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Uuid"
]
},
"nullable": []
},
"hash": "9c17ddca92e0a93051b36bb688ad6c1c24108e45c94be52595ca6cb82135f4eb"
}
backend/.sqlx/query-9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "9ebb9c16948a695a053068d9a1df0152691271ac33d8c82db47be11746ccbcef"
}
backend/.sqlx/query-ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)\n WHERE id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Jsonb",
"Uuid"
]
},
"nullable": []
},
"hash": "ad8f9e0b06f288051cbec3c91877430a1d61353b9122fd42777d92e4cbc9f4fa"
}
backend/.sqlx/query-b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76.json
Generated
+2 -2
View File
@@ -18,8 +18,8 @@
"Left": []
},
"nullable": [
false,
true
true,
false
]
},
"hash": "b3dbdfb50ee8118bdaed3164b210cb549a34b96554ae1872355b90304f5dcb76"
backend/.sqlx/query-8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb.json → backend/.sqlx/query-b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)",
"query": "INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31)",
"describe": {
"columns": [],
"parameters": {
@@ -72,10 +72,11 @@
"Varchar",
"Bool",
"Bool",
"Varchar"
"Varchar",
"Bool"
]
},
"nullable": []
},
"hash": "8e7ff45c5378c3a3406ba94dc0653afa5d28c072203617c598caefaaf1bafcfb"
"hash": "b49b1eaf58e62063c7a03039d0c36e65991b1d21a3306390fb1c7ca38babafe3"
}
backend/.sqlx/query-b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1 AND path = $2)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "b57a188ca137162a2848ebf81fa3aeca9a14ac628c61a358e2c6612c57249b0f"
}
backend/.sqlx/query-b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9.json
Generated
-17
View File
@@ -1,17 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)\n WHERE id = $4",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Jsonb",
"Jsonb",
"Uuid"
]
},
"nullable": []
},
"hash": "b64c7f3b483c74c51aecda65664c36d9686112182dbf35651e494da739b922f9"
}
backend/.sqlx/query-c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1)\n WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Uuid"
]
},
"nullable": []
},
"hash": "c14009d133956710f4435a5984c18b9517e256978512f014e1bf5c270f499772"
}
backend/.sqlx/query-c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8.json
Generated
+37
View File
@@ -0,0 +1,37 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Bool",
"Bool",
"Bool",
{
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
},
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "c229744534f17f7622c3dee21bb1e7292ff17e6dffe58e80e53bff8baade07c8"
}
backend/.sqlx/query-dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": []
},
"hash": "dc9a906d6c6156a84fccf4e3a2a7c08d8ed4984409b669162f0a1fc1aa48e188"
}
backend/.sqlx/query-f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8.json → backend/.sqlx/query-f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1",
"query": "SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1",
"describe": {
"columns": [
{
@@ -24,5 +24,5 @@
null
]
},
"hash": "f80e94176224ef23416da99b26ad5c330d65016fb93345b054cbb63a4bf37fc8"
"hash": "f9e0e35b4789a4da89f7bb21fa6b250fd64ee0e0b299471979db5ff8ee906929"
}
backend/.sqlx/query-e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99.json → backend/.sqlx/query-fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
"query": "select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where path = $1 AND workspace_id = $2 AND\n created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND\n deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
"describe": {
"columns": [
{
@@ -82,6 +82,11 @@
"ordinal": 10,
"name": "timeout",
"type_info": "Int4"
},
{
"ordinal": 11,
"name": "has_preprocessor",
"type_info": "Bool"
}
],
"parameters": {
@@ -101,8 +106,9 @@
true,
true,
true,
true,
true
]
},
"hash": "e1c715020f1efb00123171edfb6173bfeaaa95d4f870240e42b76e5df304ce99"
"hash": "fc7ac95d8d5f4532a2b0e50f84c1e273d25543401639a68f868e1e4fd8b0640a"
}
backend/.sqlx/query-fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183.json
Generated
+87
View File
@@ -0,0 +1,87 @@
{
"db_name": "PostgreSQL",
"query": "SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as \"http_method: _\" FROM http_trigger",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "script_path",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "is_flow",
"type_info": "Bool"
},
{
"ordinal": 3,
"name": "route_path",
"type_info": "Varchar"
},
{
"ordinal": 4,
"name": "workspace_id",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "is_async",
"type_info": "Bool"
},
{
"ordinal": 6,
"name": "requires_auth",
"type_info": "Bool"
},
{
"ordinal": 7,
"name": "edited_by",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "email",
"type_info": "Varchar"
},
{
"ordinal": 9,
"name": "http_method: _",
"type_info": {
"Custom": {
"name": "http_method",
"kind": {
"Enum": [
"get",
"post",
"put",
"delete",
"patch"
]
}
}
}
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
false,
false,
false,
false,
false,
false,
false,
false,
false
]
},
"hash": "fe4f91ca7e179e58041a5c0b1a36015175e9301c01a99e8f2b82cb853349a183"
}
backend/Cargo.lock
Generated
+56 -50
View File
@@ -1056,9 +1056,9 @@ dependencies = [
[[package]]
name = "axum"
version = "0.7.5"
version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf"
checksum = "8f43644eed690f5374f1af436ecd6aea01cd201f6fbdf0178adaf6907afb2cec"
dependencies = [
"async-trait",
"axum-core",
@@ -1083,7 +1083,7 @@ dependencies = [
"serde_urlencoded",
"sync_wrapper 1.0.1",
"tokio",
"tower 0.4.13",
"tower 0.5.1",
"tower-layer",
"tower-service",
"tracing",
@@ -1091,9 +1091,9 @@ dependencies = [
[[package]]
name = "axum-core"
version = "0.4.3"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3"
checksum = "5e6b8ba012a258d63c9adfa28b9ddcf66149da6f986c5b5452e629d5ee64bf00"
dependencies = [
"async-trait",
"bytes",
@@ -1104,7 +1104,7 @@ dependencies = [
"mime",
"pin-project-lite",
"rustversion",
"sync_wrapper 0.1.2",
"sync_wrapper 1.0.1",
"tower-layer",
"tower-service",
"tracing",
@@ -1735,9 +1735,9 @@ dependencies = [
[[package]]
name = "clap"
version = "4.5.17"
version = "4.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e5a21b8495e732f1b3c364c9949b201ca7bae518c502c80256c96ad79eaf6ac"
checksum = "b0956a43b323ac1afaffc053ed5c4b7c1f1800bacd1683c353aabbb752515dd3"
dependencies = [
"clap_builder",
"clap_derive",
@@ -1745,9 +1745,9 @@ dependencies = [
[[package]]
name = "clap_builder"
version = "4.5.17"
version = "4.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8cf2dd12af7a047ad9d6da2b6b249759a22a7abc0f474c1dae1777afa4b21a73"
checksum = "4d72166dd41634086d5803a47eb71ae740e61d84709c36f3c34110173db3961b"
dependencies = [
"anstream",
"anstyle",
@@ -1757,9 +1757,9 @@ dependencies = [
[[package]]
name = "clap_derive"
version = "4.5.13"
version = "4.5.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
checksum = "4ac6a0c7b1a9e9a5186361f67dfa1b88213572f427fb9ab038efb2bd8c582dab"
dependencies = [
"heck 0.5.0",
"proc-macro2",
@@ -5442,7 +5442,7 @@ dependencies = [
"md-5 0.10.6",
"parking_lot",
"percent-encoding",
"quick-xml 0.36.1",
"quick-xml 0.36.2",
"rand 0.8.5",
"reqwest 0.12.7",
"ring 0.17.8",
@@ -5991,9 +5991,9 @@ dependencies = [
[[package]]
name = "pkg-config"
version = "0.3.30"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
[[package]]
name = "polling"
@@ -6012,9 +6012,9 @@ dependencies = [
[[package]]
name = "portable-atomic"
version = "1.7.0"
version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce"
[[package]]
name = "postgres-native-tls"
@@ -6374,9 +6374,9 @@ dependencies = [
[[package]]
name = "quick-xml"
version = "0.36.1"
version = "0.36.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc"
checksum = "f7649a7b4df05aed9ea7ec6f628c67c9953a43869b8bc50929569b2999d443fe"
dependencies = [
"memchr",
"serde",
@@ -7415,9 +7415,9 @@ dependencies = [
[[package]]
name = "security-framework-sys"
version = "2.11.1"
version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf"
checksum = "ea4a292869320c0272d7bc55a5a6aafaff59b4f63404a003887b679a2e05b4b6"
dependencies = [
"core-foundation-sys",
"libc",
@@ -7755,9 +7755,9 @@ dependencies = [
[[package]]
name = "simdutf8"
version = "0.1.4"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
[[package]]
name = "simple_asn1"
@@ -9061,18 +9061,18 @@ dependencies = [
[[package]]
name = "thiserror"
version = "1.0.63"
version = "1.0.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
checksum = "d50af8abc119fb8bb6dbabcfa89656f46f84aa0ac7688088608076ad2b459a84"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.63"
version = "1.0.64"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
checksum = "08904e7672f5eb876eaaf87e0ce17857500934f4981c4a0ab2b4aa98baac7fc3"
dependencies = [
"proc-macro2",
"quote",
@@ -9497,7 +9497,6 @@ dependencies = [
"tokio",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
@@ -9506,8 +9505,14 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f"
dependencies = [
"futures-core",
"futures-util",
"pin-project-lite",
"sync_wrapper 0.1.2",
"tokio",
"tower-layer",
"tower-service",
"tracing",
]
[[package]]
@@ -10421,7 +10426,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "windmill"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"axum",
@@ -10463,7 +10468,7 @@ dependencies = [
[[package]]
name = "windmill-api"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"argon2",
@@ -10497,6 +10502,7 @@ dependencies = [
"lazy_static",
"magic-crypt",
"mail-parser",
"matchit",
"mime_guess",
"native-tls",
"object_store",
@@ -10547,7 +10553,7 @@ dependencies = [
[[package]]
name = "windmill-api-client"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"base64 0.21.7",
"chrono",
@@ -10565,7 +10571,7 @@ dependencies = [
[[package]]
name = "windmill-audit"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"chrono",
"serde",
@@ -10578,7 +10584,7 @@ dependencies = [
[[package]]
name = "windmill-common"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"async-stream",
@@ -10622,7 +10628,7 @@ dependencies = [
[[package]]
name = "windmill-git-sync"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"regex",
"rsmq_async",
@@ -10637,7 +10643,7 @@ dependencies = [
[[package]]
name = "windmill-indexer"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"bytes",
@@ -10658,7 +10664,7 @@ dependencies = [
[[package]]
name = "windmill-parser"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"convert_case 0.6.0",
"serde",
@@ -10667,7 +10673,7 @@ dependencies = [
[[package]]
name = "windmill-parser-bash"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -10679,7 +10685,7 @@ dependencies = [
[[package]]
name = "windmill-parser-go"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"gosyn",
@@ -10691,7 +10697,7 @@ dependencies = [
[[package]]
name = "windmill-parser-graphql"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -10703,7 +10709,7 @@ dependencies = [
[[package]]
name = "windmill-parser-php"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"itertools 0.13.0",
@@ -10714,7 +10720,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"itertools 0.13.0",
@@ -10725,7 +10731,7 @@ dependencies = [
[[package]]
name = "windmill-parser-py-imports"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -10743,7 +10749,7 @@ dependencies = [
[[package]]
name = "windmill-parser-rust"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"convert_case 0.6.0",
@@ -10760,7 +10766,7 @@ dependencies = [
[[package]]
name = "windmill-parser-sql"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -10772,7 +10778,7 @@ dependencies = [
[[package]]
name = "windmill-parser-ts"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"lazy_static",
@@ -10790,7 +10796,7 @@ dependencies = [
[[package]]
name = "windmill-parser-wasm"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"getrandom 0.2.15",
@@ -10811,7 +10817,7 @@ dependencies = [
[[package]]
name = "windmill-parser-yaml"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"serde_json",
@@ -10821,7 +10827,7 @@ dependencies = [
[[package]]
name = "windmill-queue"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"async-recursion",
@@ -10854,7 +10860,7 @@ dependencies = [
[[package]]
name = "windmill-sql-datatype-parser-wasm"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"wasm-bindgen",
"wasm-bindgen-test",
@@ -10864,7 +10870,7 @@ dependencies = [
[[package]]
name = "windmill-worker"
version = "1.397.1"
version = "1.398.1"
dependencies = [
"anyhow",
"async-recursion",
backend/Cargo.toml
+3 -2
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.397.1"
version = "1.398.1"
authors.workspace = true
edition.workspace = true
@@ -27,7 +27,7 @@ members = [
]
[workspace.package]
version = "1.397.1"
version = "1.398.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -252,6 +252,7 @@ indexmap = { version = "2.2.5", features = ["serde"]}
tokio-native-tls = "^0"
openssl = "=0.10"
mail-parser = "^0"
matchit = "=0.7.3"
datafusion = "39.0.0"
object_store = { version = "0.10.0", features = ["aws", "azure"] }
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
534f50aae2f89281304b3c3ba943d18afc07fad4
e093d51a219ce4ff0562a02db24ec402554a1f05
backend/migrations/20240917172503_job_routing.down.sql
+7
View File
@@ -0,0 +1,7 @@
-- Add down migration script here
DROP TABLE http_trigger;
DROP TYPE http_method;
ALTER TABLE script DROP COLUMN has_preprocessor;
DROP FUNCTION prevent_route_path_change();
backend/migrations/20240917172503_job_routing.up.sql
+61
View File
@@ -0,0 +1,61 @@
-- Add up migration script here
CREATE TYPE http_method AS ENUM ('get', 'post', 'put', 'delete', 'patch');
CREATE TABLE http_trigger (
path VARCHAR(255) NOT NULL,
route_path VARCHAR(255) NOT NULL,
route_path_key VARCHAR(255) NOT NULL,
script_path VARCHAR(255) NOT NULL,
is_flow BOOLEAN NOT NULL,
workspace_id VARCHAR(50) NOT NULL,
edited_by VARCHAR(50) NOT NULL,
email VARCHAR(255) NOT NULL,
edited_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
extra_perms JSONB NOT NULL DEFAULT '{}',
is_async BOOLEAN NOT NULL DEFAULT FALSE,
requires_auth BOOLEAN NOT NULL DEFAULT FALSE,
http_method http_method NOT NULL,
PRIMARY KEY (path, workspace_id)
);
GRANT SELECT, UPDATE ON http_trigger TO windmill_user;
GRANT ALL ON http_trigger TO windmill_admin;
ALTER TABLE http_trigger ENABLE ROW LEVEL SECURITY;
CREATE POLICY see_folder_extra_perms_user_select ON http_trigger FOR SELECT TO windmill_user
USING (SPLIT_PART(http_trigger.path, '/', 1) = 'f' AND SPLIT_PART(http_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_read'), ',')::text[]));
CREATE POLICY see_folder_extra_perms_user_update ON http_trigger FOR UPDATE TO windmill_user
USING (SPLIT_PART(http_trigger.path, '/', 1) = 'f' AND SPLIT_PART(http_trigger.path, '/', 2) = any(regexp_split_to_array(current_setting('session.folders_write'), ',')::text[]));
CREATE POLICY see_extra_perms_user_select ON http_trigger FOR SELECT TO windmill_user
USING (extra_perms ? CONCAT('u/', current_setting('session.user')));
CREATE POLICY see_extra_perms_user_update ON http_trigger FOR UPDATE TO windmill_user
USING ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_groups_select ON http_trigger FOR SELECT TO windmill_user
USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[]);
CREATE POLICY see_extra_perms_groups_update ON http_trigger FOR UPDATE TO windmill_user
USING (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
CREATE OR REPLACE FUNCTION prevent_route_path_change()
RETURNS TRIGGER AS $$
BEGIN
IF CURRENT_USER <> 'windmill_admin' AND NEW.route_path <> OLD.route_path THEN
RAISE EXCEPTION 'Modification of route_path is only allowed by admins';
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
CREATE TRIGGER check_route_path_change
BEFORE UPDATE ON http_trigger
FOR EACH ROW
EXECUTE FUNCTION prevent_route_path_change();
ALTER TABLE script ADD COLUMN has_preprocessor BOOLEAN;
backend/migrations/20240919200009_migrate_smtp_settings.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20240919200009_migrate_smtp_settings.up.sql
+8
View File
@@ -0,0 +1,8 @@
-- Add up migration script here
INSERT INTO global_settings (name, value)
SELECT 'smtp_settings', config
FROM config
WHERE name = 'server'
ON CONFLICT (name) DO UPDATE SET value = excluded.value;
DELETE FROM config WHERE name = 'server';
backend/parsers/windmill-parser-bash/src/lib.rs
+16 -3
View File
@@ -16,7 +16,13 @@ pub fn parse_bash_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_bash_file(&code)?;
if let Some(x) = parsed {
let args = x;
Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args,
no_main_func: None,
has_preprocessor: None,
})
} else {
Err(anyhow!("Error parsing bash script".to_string()))
}
@@ -26,7 +32,13 @@ pub fn parse_powershell_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_powershell_file(&code)?;
if let Some(x) = parsed {
let args = x;
Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args,
no_main_func: None,
has_preprocessor: None,
})
} else {
Err(anyhow!("Error parsing powershell script".to_string()))
}
@@ -173,7 +185,8 @@ non_required="${5:-}"
oidx: None
}
],
no_main_func: None
no_main_func: None,
has_preprocessor: None
}
);
backend/parsers/windmill-parser-go/src/lib.rs
+4 -1
View File
@@ -42,6 +42,7 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result<MainArgSignature> {
star_kwargs: false,
args,
no_main_func: Some(false),
has_preprocessor: None,
})
} else {
Ok(MainArgSignature {
@@ -49,6 +50,7 @@ pub fn parse_go_sig(code: &str) -> anyhow::Result<MainArgSignature> {
star_kwargs: false,
args: vec![],
no_main_func: Some(true),
has_preprocessor: None,
})
}
}
@@ -241,7 +243,8 @@ func main(x int, y string, z bool, l []string, o struct { Name string `json:"nam
oidx: None
},
],
no_main_func: Some(false)
no_main_func: Some(false),
has_preprocessor: None
}
);
backend/parsers/windmill-parser-graphql/src/lib.rs
+9 -2
View File
@@ -15,7 +15,13 @@ pub fn parse_graphql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_graphql_file(&code)?;
if let Some(x) = parsed {
let args = x;
Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args,
no_main_func: None,
has_preprocessor: None,
})
} else {
Err(anyhow!("Error parsing sql".to_string()))
}
@@ -119,7 +125,8 @@ query($i: Int, $arr: [String]!, $wahoo: String = "wahoo") {
oidx: None
}
],
no_main_func: None
no_main_func: None,
has_preprocessor: None
}
);
backend/parsers/windmill-parser-php/src/lib.rs
+4 -1
View File
@@ -82,6 +82,7 @@ pub fn parse_php_signature(
star_kwargs: false,
args,
no_main_func: Some(false),
has_preprocessor: None,
})
} else {
Ok(MainArgSignature {
@@ -89,6 +90,7 @@ pub fn parse_php_signature(
star_kwargs: false,
args: vec![],
no_main_func: Some(true),
has_preprocessor: None,
})
}
}
@@ -159,7 +161,8 @@ function main(string $input1 = \"hey\", bool $input2 = false, int $input3 = 3, f
oidx: None
}
],
no_main_func: Some(false)
no_main_func: Some(false),
has_preprocessor: None
}
);
backend/parsers/windmill-parser-py/src/lib.rs
+68 -1
View File
@@ -63,9 +63,17 @@ pub fn parse_python_signature(
) -> anyhow::Result<MainArgSignature> {
let main_name = override_main.unwrap_or("main".to_string());
let has_preprocessor = !filter_non_main(code, "preprocessor").is_empty();
let filtered_code = filter_non_main(code, &main_name);
if filtered_code.is_empty() {
return Err(anyhow::anyhow!("No main function found".to_string(),));
return Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![],
no_main_func: Some(true),
has_preprocessor: Some(has_preprocessor),
});
}
let ast = Suite::parse(&filtered_code, "main.py")
.map_err(|e| anyhow::anyhow!("Error parsing code: {}", e.to_string()))?;
@@ -119,6 +127,7 @@ pub fn parse_python_signature(
})
.collect(),
no_main_func: Some(false),
has_preprocessor: Some(has_preprocessor),
})
} else {
Ok(MainArgSignature {
@@ -126,6 +135,7 @@ pub fn parse_python_signature(
star_kwargs: false,
args: vec![],
no_main_func: Some(true),
has_preprocessor: Some(has_preprocessor),
})
}
}
@@ -325,6 +335,7 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt
},
],
no_main_func: Some(false),
has_preprocessor: Some(false)
}
);
@@ -389,6 +400,7 @@ def main(test1: str,
}
],
no_main_func: Some(false),
has_preprocessor: Some(false)
}
);
@@ -448,6 +460,7 @@ def main(test1: str,
}
],
no_main_func: Some(false),
has_preprocessor: Some(false)
}
);
@@ -491,6 +504,7 @@ def main(test1: Literal["foo", "bar"], test2: List[Literal["foo", "bar"]]): retu
}
],
no_main_func: Some(false),
has_preprocessor: Some(false)
}
);
@@ -521,6 +535,59 @@ def main(test1: DynSelect_foo): return
oidx: None
}],
no_main_func: Some(false),
has_preprocessor: Some(false)
}
);
Ok(())
}
#[test]
fn test_parse_python_sig_6() -> anyhow::Result<()> {
let code = r#"
import os
def hello(): return
"#;
//println!("{}", serde_json::to_string()?);
assert_eq!(
parse_python_signature(code, None)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![],
no_main_func: Some(true),
has_preprocessor: Some(false)
}
);
Ok(())
}
#[test]
fn test_parse_python_sig_7() -> anyhow::Result<()> {
let code = r#"
import os
def preprocessor(): return
def main(): return
"#;
//println!("{}", serde_json::to_string()?);
assert_eq!(
parse_python_signature(code, None)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![],
no_main_func: Some(false),
has_preprocessor: Some(true)
}
);
backend/parsers/windmill-parser-rust/src/lib.rs
+3 -4
View File
@@ -29,6 +29,7 @@ pub fn parse_rust_signature(code: &str) -> anyhow::Result<MainArgSignature> {
star_kwargs: false,
args,
no_main_func: Some(false),
has_preprocessor: None,
})
} else {
Ok(MainArgSignature {
@@ -36,6 +37,7 @@ pub fn parse_rust_signature(code: &str) -> anyhow::Result<MainArgSignature> {
star_kwargs: false,
args: vec![],
no_main_func: Some(true),
has_preprocessor: None,
})
}
}
@@ -455,10 +457,7 @@ fn main(
assert_eq!(ret.args[2].typ, Typ::Str(None));
assert_eq!(ret.args[3].name, "my_string_vec");
assert_eq!(
ret.args[3].otyp,
Some("Vec < String >".to_string())
);
assert_eq!(ret.args[3].otyp, Some("Vec < String >".to_string()));
assert_eq!(ret.args[3].typ, Typ::List(Box::new(Typ::Str(None))));
}
backend/parsers/windmill-parser-sql/src/lib.rs
+49 -12
View File
@@ -20,7 +20,13 @@ pub fn parse_mysql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_mysql_file(&code)?;
if let Some(x) = parsed {
let args = x;
Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args,
no_main_func: None,
has_preprocessor: None,
})
} else {
Err(anyhow!("Error parsing sql".to_string()))
}
@@ -30,7 +36,13 @@ pub fn parse_pgsql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_pg_file(&code)?;
if let Some(x) = parsed {
let args = x;
Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args,
no_main_func: None,
has_preprocessor: None,
})
} else {
Err(anyhow!("Error parsing sql".to_string()))
}
@@ -40,7 +52,13 @@ pub fn parse_bigquery_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_bigquery_file(&code)?;
if let Some(x) = parsed {
let args = x;
Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args,
no_main_func: None,
has_preprocessor: None,
})
} else {
Err(anyhow!("Error parsing sql".to_string()))
}
@@ -50,7 +68,13 @@ pub fn parse_snowflake_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_snowflake_file(&code)?;
if let Some(x) = parsed {
let args = x;
Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args,
no_main_func: None,
has_preprocessor: None,
})
} else {
Err(anyhow!("Error parsing sql".to_string()))
}
@@ -60,7 +84,13 @@ pub fn parse_mssql_sig(code: &str) -> anyhow::Result<MainArgSignature> {
let parsed = parse_mssql_file(&code)?;
if let Some(x) = parsed {
let args = x;
Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args,
no_main_func: None,
has_preprocessor: None,
})
} else {
Err(anyhow!("Error parsing sql".to_string()))
}
@@ -570,7 +600,8 @@ SELECT * FROM table WHERE token=$1::TEXT AND image=$2::BIGINT
oidx: Some(2),
},
],
no_main_func: None
no_main_func: None,
has_preprocessor: None
}
);
@@ -618,7 +649,8 @@ SELECT $2::TEXT;
oidx: Some(3),
},
],
no_main_func: None
no_main_func: None,
has_preprocessor: None
}
);
@@ -744,7 +776,8 @@ SELECT ?, ?;
oidx: None,
},
],
no_main_func: None
no_main_func: None,
has_preprocessor: None
}
);
@@ -791,7 +824,8 @@ SELECT :param2;
oidx: None,
},
],
no_main_func: None
no_main_func: None,
has_preprocessor: None
}
);
@@ -830,7 +864,8 @@ SELECT @token;
oidx: None,
},
],
no_main_func: None
no_main_func: None,
has_preprocessor: None
}
);
@@ -877,7 +912,8 @@ SELECT ?;
oidx: None,
}
],
no_main_func: None
no_main_func: None,
has_preprocessor: None
}
);
@@ -924,7 +960,8 @@ SELECT @p2;
oidx: None,
},
],
no_main_func: None
no_main_func: None,
has_preprocessor: None
}
);
backend/parsers/windmill-parser-ts/src/lib.rs
+10
View File
@@ -159,6 +159,14 @@ pub fn parse_deno_signature(
})?
.body;
let has_preprocessor = ast.iter().any(|x| match x {
ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
decl: Decl::Fn(FnDecl { ident: Ident { sym, .. }, .. }),
..
})) => &sym.to_string() == "preprocessor",
_ => false,
});
let main_name = main_override.unwrap_or("main".to_string());
let params = ast.into_iter().find_map(|x| match x {
ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(ExportDecl {
@@ -178,6 +186,7 @@ pub fn parse_deno_signature(
.map(|x| parse_param(x, &cm, skip_dflt, &mut c))
.collect::<anyhow::Result<Vec<Arg>>>()?,
no_main_func: Some(false),
has_preprocessor: Some(has_preprocessor),
};
Ok(r)
} else {
@@ -186,6 +195,7 @@ pub fn parse_deno_signature(
star_kwargs: false,
args: vec![],
no_main_func: Some(true),
has_preprocessor: Some(has_preprocessor),
})
}
}
backend/parsers/windmill-parser-wasm/build-deno-pkgs.sh → backend/parsers/windmill-parser-wasm/build-pkgs-mac.sh
+12 -6
View File
@@ -10,35 +10,41 @@ wasm-pack build --release --target web --out-dir $OUT_DIR --all-features \
OUT_DIR="pkg-ts"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "ts-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-ts"/' $OUT_DIR/package.json
sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-ts"/' $OUT_DIR/package.json
# sql languages, graphql and bash/powershell, since they all use regex
OUT_DIR="pkg-regex"
wasm-pack build --release --target web --out-dir $OUT_DIR \
--features "sql-parser,graphql-parser,bash-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-regex"/' $OUT_DIR/package.json
sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-regex"/' $OUT_DIR/package.json
# python
OUT_DIR="pkg-py"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "py-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-py"/' $OUT_DIR/package.json
sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-py"/' $OUT_DIR/package.json
# go
OUT_DIR="pkg-go"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "go-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-go"/' $OUT_DIR/package.json
sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-go"/' $OUT_DIR/package.json
# php
OUT_DIR="pkg-php"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "php-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json
sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-php"/' $OUT_DIR/package.json
# rust
OUT_DIR="pkg-rust"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "rust-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json
sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-rust"/' $OUT_DIR/package.json
# ansible
OUT_DIR="pkg-yaml"
wasm-pack build --release --target web --out-dir $OUT_DIR --features "ansible-parser" \
-Z build-std=panic_abort,std -Z build-std-features=panic_immediate_abort
sed -i '' 's/"windmill-parser-wasm"/"windmill-parser-wasm-yaml"/' $OUT_DIR/package.json
backend/parsers/windmill-parser-wasm/build.sh
+1 -1
View File
@@ -1,4 +1,4 @@
#!/bin/bash
set -eou pipefail
deno task wasmbuild --out ../../../cli/wasm/ -p windmill-parser-wasm
deno task wasmbuild --out ../../../cli/wasm/ -p windmill-parser-wasm --all-features
backend/parsers/windmill-parser-wasm/src/lib.rs
+11 -4
View File
@@ -17,8 +17,12 @@ fn wrap_sig(r: anyhow::Result<MainArgSignature>) -> String {
#[cfg(feature = "ts-parser")]
#[wasm_bindgen]
pub fn parse_deno(code: &str) -> String {
wrap_sig(windmill_parser_ts::parse_deno_signature(code, false, None))
pub fn parse_deno(code: &str, main_override: Option<String>) -> String {
wrap_sig(windmill_parser_ts::parse_deno_signature(
code,
false,
main_override,
))
}
#[cfg(feature = "ts-parser")]
@@ -65,8 +69,11 @@ pub fn parse_go(code: &str) -> String {
#[cfg(feature = "py-parser")]
#[wasm_bindgen]
pub fn parse_python(code: &str) -> String {
wrap_sig(windmill_parser_py::parse_python_signature(code, None))
pub fn parse_python(code: &str, main_override: Option<String>) -> String {
wrap_sig(windmill_parser_py::parse_python_signature(
code,
main_override,
))
}
#[cfg(feature = "sql-parser")]
backend/parsers/windmill-parser-wasm/tests/wasm.rs
+9 -4
View File
@@ -139,7 +139,8 @@ export function main(test1?: string, test2: string = \"burkina\",
oidx: None
}
],
no_main_func: Some(false)
no_main_func: Some(false),
has_preprocessor: Some(false)
}
);
@@ -215,7 +216,8 @@ export function main(test2 = \"burkina\",
oidx: None
}
],
no_main_func: Some(false)
no_main_func: Some(false),
has_preprocessor: Some(false)
}
);
@@ -264,7 +266,8 @@ export function main(foo: FooBar, {a, b}: FooBar, {c, d}: FooBar = {a: \"foo\",
oidx: None
}
],
no_main_func: Some(false)
no_main_func: Some(false),
has_preprocessor: Some(false)
}
);
@@ -294,7 +297,8 @@ export function main(foo: (\"foo\" | \"bar\")[]) {
has_default: false,
oidx: None
}],
no_main_func: Some(false)
no_main_func: Some(false),
has_preprocessor: Some(false)
}
);
@@ -434,6 +438,7 @@ Write-Output 'Testing...'
}
],
no_main_func: None,
has_preprocessor: None
}
);
backend/parsers/windmill-parser-yaml/src/lib.rs
+18 -5
View File
@@ -13,6 +13,7 @@ pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result<MainArgSignature
star_kwargs: false,
args: vec![],
no_main_func: None,
has_preprocessor: None,
});
}
@@ -55,7 +56,13 @@ pub fn parse_ansible_sig(inner_content: &str) -> anyhow::Result<MainArgSignature
}
}
}
Ok(MainArgSignature { star_args: false, star_kwargs: false, args, no_main_func: None })
Ok(MainArgSignature {
star_args: false,
star_kwargs: false,
args,
no_main_func: None,
has_preprocessor: None,
})
}
fn parse_ansible_typ(arg: &Yaml) -> Typ {
@@ -173,7 +180,12 @@ fn parse_inventories(inventory_yaml: &Yaml) -> anyhow::Result<Vec<AnsibleInvento
.and_then(|v| v.as_str())
.map(|s| s.to_string());
ret.push(AnsibleInventory { default: windmill_path, name, resource_type, pinned_resource: pin });
ret.push(AnsibleInventory {
default: windmill_path,
name,
resource_type,
pinned_resource: pin,
});
}
}
return Ok(ret);
@@ -250,8 +262,7 @@ pub fn parse_ansible_reqs(
fn parse_file_resource(yaml: &Yaml) -> anyhow::Result<FileResource> {
if let Yaml::Hash(f) = yaml {
if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("resource".to_string()))
{
if let Some(Yaml::String(resource_path)) = f.get(&Yaml::String("resource".to_string())) {
let target_path = f
.get(&Yaml::String("target".to_string()))
.and_then(|x| x.as_str())
@@ -262,7 +273,9 @@ fn parse_file_resource(yaml: &Yaml) -> anyhow::Result<FileResource> {
))?;
return Ok(FileResource { resource_path: resource_path.clone(), target_path });
}
return Err(anyhow!("File resource should have a `resource` field, linking to a text file resource"));
return Err(anyhow!(
"File resource should have a `resource` field, linking to a text file resource"
));
}
return Err(anyhow!("Invalid file resource: Should be a dictionary."));
}
backend/parsers/windmill-parser/src/lib.rs
+5 -1
View File
@@ -16,6 +16,7 @@ pub struct MainArgSignature {
pub star_kwargs: bool,
pub args: Vec<Arg>,
pub no_main_func: Option<bool>,
pub has_preprocessor: Option<bool>,
}
#[derive(Serialize, Clone, Debug, PartialEq)]
@@ -153,7 +154,10 @@ mod test {
#[test]
fn test_mixed_delimiters() {
assert_eq!(to_snake_case("test-Case_with Spaces"), "test_case_with_spaces");
assert_eq!(
to_snake_case("test-Case_with Spaces"),
"test_case_with_spaces"
);
}
#[test]
backend/src/main.rs
+14 -7
View File
@@ -8,7 +8,10 @@
use anyhow::Context;
use git_version::git_version;
use monitor::{send_current_log_file_to_object_store, send_logs_to_object_store};
use monitor::{
reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
send_logs_to_object_store,
};
use rand::Rng;
use sqlx::{postgres::PgListener, Pool, Postgres};
use std::{
@@ -35,7 +38,7 @@ use windmill_common::{
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING,
OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
},
scripts::ScriptLang,
stats_ee::schedule_stats,
@@ -74,7 +77,7 @@ use crate::monitor::{
reload_critical_error_channels_setting, reload_extra_pip_index_url_setting,
reload_hub_base_url_setting, reload_job_default_timeout_setting, reload_jwt_secret_setting,
reload_license_key, reload_npm_config_registry_setting, reload_pip_index_url_setting,
reload_retention_period_setting, reload_scim_token_setting, reload_server_config,
reload_retention_period_setting, reload_scim_token_setting, reload_smtp_config,
reload_worker_config,
};
@@ -173,7 +176,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
None,
&job_dir,
"cache_init",
windmill_worker::get_common_bun_proc_envs("").await,
windmill_worker::get_common_bun_proc_envs(None).await,
false,
)
.await?;
@@ -563,9 +566,7 @@ Windmill Community Edition {GIT_VERSION}
"notify_config_change" => {
match n.payload() {
"server" if server_mode => {
tracing::info!("Server config change detected: {}", n.payload());
reload_server_config(&db).await;
tracing::error!("Server config change detected but server config is obsolete: {}", n.payload());
},
a@ _ if worker_mode && a == format!("worker__{}", *WORKER_GROUP) => {
tracing::info!("Worker config change detected: {}", n.payload());
@@ -609,6 +610,12 @@ Windmill Community Edition {GIT_VERSION}
tracing::error!("Error loading default tag per workspace workspaces: {e:#}");
}
}
SMTP_SETTING => {
reload_smtp_config(&db).await;
},
TIMEOUT_WAIT_RESULT_SETTING => {
reload_timeout_wait_result_setting(&db).await
},
RETENTION_PERIOD_SECS_SETTING => {
reload_retention_period_setting(&db).await
},
backend/src/monitor.rs
+22 -11
View File
@@ -22,6 +22,7 @@ use tokio::{
#[cfg(feature = "embedding")]
use windmill_api::embeddings::update_embeddings_db;
use windmill_api::{
jobs::TIMEOUT_WAIT_RESULT,
oauth2_ee::{build_oauth_clients, OAuthClient},
DEFAULT_BODY_LIMIT, IS_SECURE, OAUTH_CLIENTS, REQUEST_SIZE_LIMIT, SAML_METADATA, SCIM_TOKEN,
};
@@ -40,17 +41,17 @@ use windmill_common::{
KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
},
jobs::QueuedJob,
oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH,
server::load_server_config,
server::load_smtp_config,
tracing_init::JSON_FMT,
users::truncate_token,
utils::{now_from_db, rd_string, report_critical_error, Mode},
worker::{
load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting,
DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SERVER_CONFIG, WORKER_CONFIG,
DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, SMTP_CONFIG, WORKER_CONFIG,
WORKER_GROUP,
},
BASE_URL, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS,
@@ -172,8 +173,9 @@ pub async fn initial_load(
reload_s3_cache_setting(&db).await;
}
reload_smtp_config(&db).await;
if server_mode {
reload_server_config(&db).await;
reload_retention_period_setting(&db).await;
reload_request_size(&db).await;
reload_saml_metadata_setting(&db).await;
@@ -698,6 +700,15 @@ pub async fn reload_scim_token_setting(db: &DB) {
.await;
}
pub async fn reload_timeout_wait_result_setting(db: &DB) {
reload_option_setting_with_tracing(
db,
TIMEOUT_WAIT_RESULT_SETTING,
"TIMEOUT_WAIT_RESULT",
TIMEOUT_WAIT_RESULT.clone(),
)
.await;
}
pub async fn reload_saml_metadata_setting(db: &DB) {
reload_option_setting_with_tracing(
db,
@@ -1129,14 +1140,14 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
}
}
pub async fn reload_server_config(db: &Pool<Postgres>) {
let config = load_server_config(&db).await;
if let Err(e) = config {
tracing::error!("Error reloading server config: {:?}", e)
pub async fn reload_smtp_config(db: &Pool<Postgres>) {
let smtp_config = load_smtp_config(&db).await;
if let Err(e) = smtp_config {
tracing::error!("Error reloading smtp config: {:?}", e)
} else {
let mut wc = SERVER_CONFIG.write().await;
tracing::info!("Reloading server config...");
*wc = config.unwrap()
let mut wc = SMTP_CONFIG.write().await;
tracing::info!("Reloading smtp config...");
*wc = smtp_config.unwrap()
}
}
backend/tests/worker.rs
+2
View File
@@ -3294,6 +3294,7 @@ async fn run_deployed_relative_imports(
visible_to_runner_only: None,
no_main_func: None,
codebase: None,
has_preprocessor: None,
},
)
.await
@@ -3324,6 +3325,7 @@ async fn run_deployed_relative_imports(
dedicated_worker: None,
language,
priority: None,
apply_preprocessor: false,
})
.push(&db2)
.await;
backend/windmill-api/Cargo.toml
+1
View File
@@ -94,6 +94,7 @@ object_store = { workspace = true, optional = true}
openidconnect = { workspace = true, optional = true}
url = { workspace = true, optional = true}
jsonwebtoken = { workspace = true }
matchit.workspace = true
pin-project.workspace = true
crc.workspace = true
backend/windmill-api/openapi.yaml
+357 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.397.1
version: 1.398.1
title: Windmill API
contact:
@@ -742,6 +742,35 @@ paths:
schema:
type: string
/settings/test_critical_channels:
post:
summary: test critical channels
operationId: testCriticalChannels
tags:
- setting
requestBody:
description: test critical channel payload
required: true
content:
application/json:
schema:
type: array
items:
type: object
properties:
email:
type: string
slack_channel:
type: string
responses:
"200":
description: status
content:
text/plain:
schema:
type: string
/settings/test_license_key:
post:
summary: test license key
@@ -4182,6 +4211,11 @@ paths:
in: query
schema:
type: integer
- name: skip_preprocessor
description: skip the preprocessor
in: query
schema:
type: boolean
- $ref: "#/components/parameters/ParentJob"
- $ref: "#/components/parameters/WorkerTag"
- $ref: "#/components/parameters/CacheTtl"
@@ -5328,6 +5362,11 @@ paths:
in: query
schema:
type: integer
- name: skip_preprocessor
description: skip the preprocessor
in: query
schema:
type: boolean
- $ref: "#/components/parameters/ParentJob"
- $ref: "#/components/parameters/WorkerTag"
- $ref: "#/components/parameters/NewJobId"
@@ -5436,6 +5475,11 @@ paths:
in: query
schema:
type: integer
- name: skip_preprocessor
description: skip the preprocessor
in: query
schema:
type: boolean
- $ref: "#/components/parameters/ParentJob"
- $ref: "#/components/parameters/WorkerTag"
- $ref: "#/components/parameters/CacheTtl"
@@ -6822,6 +6866,190 @@ paths:
"201":
description: default error handler set
/w/{workspace}/http_triggers/create:
post:
summary: create http trigger
operationId: createHttpTrigger
tags:
- http_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: new http trigger
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/NewHttpTrigger"
responses:
"201":
description: http trigger created
content:
text/plain:
schema:
type: string
/w/{workspace}/http_triggers/update/{path}:
post:
summary: update http trigger
operationId: updateHttpTrigger
tags:
- http_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
requestBody:
description: updated trigger
required: true
content:
application/json:
schema:
$ref: "#/components/schemas/EditHttpTrigger"
responses:
"200":
description: http trigger updated
content:
text/plain:
schema:
type: string
/w/{workspace}/http_triggers/delete/{path}:
delete:
summary: delete http trigger
operationId: deleteHttpTrigger
tags:
- http_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: http trigger deleted
content:
text/plain:
schema:
type: string
/w/{workspace}/http_triggers/get/{path}:
get:
summary: get http trigger
operationId: getHttpTrigger
tags:
- http_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: http trigger deleted
content:
application/json:
schema:
$ref: "#/components/schemas/HttpTrigger"
/w/{workspace}/http_triggers/list:
get:
summary: list http triggers
operationId: listHttpTriggers
tags:
- http_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
required: true
- $ref: "#/components/parameters/Page"
- $ref: "#/components/parameters/PerPage"
- name: path
description: filter by path
in: query
schema:
type: string
- name: is_flow
in: query
schema:
type: boolean
- name: path_start
in: query
schema:
type: string
responses:
"200":
description: http trigger list
content:
application/json:
schema:
type: array
items:
$ref: "#/components/schemas/HttpTrigger"
/w/{workspace}/http_triggers/exists/{path}:
get:
summary: does http trigger exists
operationId: existsHttpTrigger
tags:
- http_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/Path"
responses:
"200":
description: http trigger exists
content:
application/json:
schema:
type: boolean
/w/{workspace}/http_triggers/route_exists:
post:
summary: does route exists
operationId: existsRoute
tags:
- http_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
requestBody:
description: route exists request
required: true
content:
application/json:
schema:
type: object
properties:
route_path:
type: string
http_method:
type: string
enum: ["get", "post", "put", "delete", "patch"]
required:
- kind
- route_path
- http_method
responses:
"200":
description: route exists
content:
application/json:
schema:
type: boolean
/w/{workspace}/http_triggers/used:
get:
summary: whether http triggers are used
operationId: used
tags:
- http_trigger
parameters:
- $ref: "#/components/parameters/WorkspaceId"
responses:
"200":
description: whether http triggers are used
content:
application/json:
schema:
type: boolean
/groups/list:
get:
summary: list instance groups
@@ -7648,6 +7876,7 @@ paths:
folder,
app,
raw_app,
http_trigger,
]
responses:
"200":
@@ -7684,6 +7913,7 @@ paths:
folder,
app,
raw_app,
http_trigger,
]
requestBody:
description: acl to add
@@ -7731,6 +7961,7 @@ paths:
folder,
app,
raw_app,
http_trigger,
]
requestBody:
description: acl to add
@@ -9421,6 +9652,8 @@ components:
type: boolean
codebase:
type: string
has_preprocessor:
type: boolean
required:
- hash
@@ -9438,6 +9671,7 @@ components:
- kind
- starred
- no_main_func
- has_preprocessor
NewScript:
@@ -9519,6 +9753,8 @@ components:
type: boolean
codebase:
type: string
has_preprocessor:
type: boolean
required:
- path
- summary
@@ -10319,6 +10555,9 @@ components:
no_main_func:
type: boolean
nullable: true
has_preprocessor:
type: boolean
nullable: true
required:
- star_args
- start_kwargs
@@ -10326,6 +10565,7 @@ components:
- type
- error
- no_main_func
- has_preprocessor
Preview:
type: object
@@ -10715,6 +10955,122 @@ components:
- is_flow
- args
HttpTrigger:
type: object
properties:
path:
type: string
edited_by:
type: string
edited_at:
type: string
format: date-time
script_path:
type: string
route_path:
type: string
is_flow:
type: boolean
extra_perms:
type: object
additionalProperties:
type: boolean
email:
type: string
workspace_id:
type: string
http_method:
type: string
enum:
- get
- post
- put
- delete
- patch
is_async:
type: boolean
requires_auth:
type: boolean
required:
- path
- edited_by
- edited_at
- script_path
- route_path
- extra_perms
- is_flow
- email
- workspace_id
- is_async
- requires_auth
- http_method
NewHttpTrigger:
type: object
properties:
path:
type: string
script_path:
type: string
route_path:
type: string
is_flow:
type: boolean
http_method:
type: string
enum:
- get
- post
- put
- delete
- patch
is_async:
type: boolean
requires_auth:
type: boolean
required:
- path
- script_path
- route_path
- is_flow
- is_async
- requires_auth
- http_method
EditHttpTrigger:
type: object
properties:
path:
type: string
script_path:
type: string
route_path:
type: string
is_flow:
type: boolean
http_method:
type: string
enum:
- get
- post
- put
- delete
- patch
is_async:
type: boolean
requires_auth:
type: boolean
required:
- path
- script_path
- is_flow
- kind
- is_async
- requires_auth
- http_method
Group:
type: object
properties:
backend/windmill-api/src/flows.rs
+3 -1
View File
@@ -1258,6 +1258,7 @@ mod tests {
delete_after_use: None,
continue_on_error: None,
})),
preprocessor_module: None,
same_worker: false,
concurrent_limit: None,
concurrency_time_window_s: None,
@@ -1330,7 +1331,8 @@ mod tests {
"expr": "previous.isEmpty()",
"skip_if_stopped": false
}
}
},
"preprocessor_module": Option::<String>::None
});
assert_eq!(dbg!(serde_json::json!(fv)), dbg!(expect));
}
backend/windmill-api/src/granular_acls.rs
+11 -2
View File
@@ -23,8 +23,17 @@ use windmill_common::{
utils::{not_found_if_none, StripPath},
};
const KINDS: [&str; 9] = [
"script", "group_", "resource", "schedule", "variable", "flow", "folder", "app", "raw_app",
const KINDS: [&str; 10] = [
"script",
"group_",
"resource",
"schedule",
"variable",
"flow",
"folder",
"app",
"raw_app",
"http_trigger",
];
pub fn workspaced_service() -> Router {
backend/windmill-api/src/http_triggers.rs
+651
View File
@@ -0,0 +1,651 @@
use axum::{
extract::{Path, Query},
response::IntoResponse,
routing::{delete, get, post},
Extension, Json, Router,
};
use http::{HeaderMap, StatusCode};
use serde::{Deserialize, Serialize};
use sql_builder::{bind::Bind, SqlBuilder};
use sqlx::prelude::FromRow;
use std::collections::HashMap;
use tower_http::cors::CorsLayer;
use windmill_audit::{audit_ee::audit_log, ActionKind};
use windmill_common::{
auth::fetch_authed_from_permissioned_as,
db::UserDB,
error::{self, JsonResult},
users::username_to_permissioned_as,
utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
worker::{to_raw_value, CLOUD_HOSTED},
};
use windmill_queue::PushArgsOwned;
use crate::{
db::{ApiAuthed, DB},
jobs::{
run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_flow_by_path_internal,
run_wait_result_script_by_path_internal, RunJobQuery,
},
users::OptAuthed,
};
lazy_static::lazy_static! {
static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/:\w+").unwrap();
}
pub fn routes_global_service() -> Router {
let cors = CorsLayer::new()
.allow_methods([
http::Method::GET,
http::Method::POST,
http::Method::DELETE,
http::Method::PUT,
http::Method::PATCH,
])
.allow_headers([http::header::CONTENT_TYPE, http::header::AUTHORIZATION])
.allow_origin(tower_http::cors::Any);
Router::new()
.route(
"/*path",
get(route_job)
.post(route_job)
.delete(route_job)
.put(route_job)
.patch(route_job)
.head(|| async { "" }),
)
.layer(cors)
}
pub fn workspaced_service() -> Router {
Router::new()
.route("/create", post(create_trigger))
.route("/list", get(list_triggers))
.route("/get/*path", get(get_trigger))
.route("/update/*path", post(update_trigger))
.route("/delete/*path", delete(delete_trigger))
.route("/exists/*path", get(exists_trigger))
.route("/used", get(used))
.route("/route_exists", post(exists_route))
}
#[derive(Serialize, Deserialize, sqlx::Type)]
#[sqlx(type_name = "HTTP_METHOD", rename_all = "lowercase")]
#[serde(rename_all = "lowercase")]
enum HttpMethod {
Get,
Post,
Put,
Delete,
Patch,
}
impl From<HttpMethod> for http::Method {
fn from(method: HttpMethod) -> Self {
match method {
HttpMethod::Get => http::Method::GET,
HttpMethod::Post => http::Method::POST,
HttpMethod::Put => http::Method::PUT,
HttpMethod::Delete => http::Method::DELETE,
HttpMethod::Patch => http::Method::PATCH,
}
}
}
#[derive(Deserialize)]
struct NewTrigger {
path: String,
route_path: String,
script_path: String,
is_flow: bool,
is_async: bool,
requires_auth: bool,
http_method: HttpMethod,
}
#[derive(FromRow, Serialize)]
struct Trigger {
workspace_id: String,
path: String,
route_path: String,
route_path_key: String,
script_path: String,
is_flow: bool,
edited_by: String,
email: String,
edited_at: chrono::DateTime<chrono::Utc>,
extra_perms: serde_json::Value,
is_async: bool,
requires_auth: bool,
http_method: HttpMethod,
}
#[derive(Deserialize)]
struct EditTrigger {
path: String,
route_path: Option<String>,
script_path: String,
is_flow: bool,
is_async: bool,
requires_auth: bool,
http_method: HttpMethod,
}
#[derive(Deserialize)]
pub struct ListTriggerQuery {
pub page: Option<usize>,
pub per_page: Option<usize>,
pub path: Option<String>,
pub is_flow: Option<bool>,
pub path_start: Option<String>,
}
async fn list_triggers(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(lst): Query<ListTriggerQuery>,
) -> error::JsonResult<Vec<Trigger>> {
let mut tx = user_db.begin(&authed).await?;
let (per_page, offset) = paginate(Pagination { per_page: lst.per_page, page: lst.page });
let mut sqlb = SqlBuilder::select_from("http_trigger")
.field("*")
.order_by("edited_at", true)
.and_where("workspace_id = ?".bind(&w_id))
.offset(offset)
.limit(per_page)
.clone();
if let Some(path) = lst.path {
sqlb.and_where_eq("script_path", "?".bind(&path));
}
if let Some(is_flow) = lst.is_flow {
sqlb.and_where_eq("is_flow", "?".bind(&is_flow));
}
if let Some(path_start) = &lst.path_start {
sqlb.and_where_like_left("path", path_start);
}
let sql = sqlb
.sql()
.map_err(|e| error::Error::InternalErr(e.to_string()))?;
let rows = sqlx::query_as::<_, Trigger>(&sql)
.fetch_all(&mut *tx)
.await?;
tx.commit().await?;
Ok(Json(rows))
}
async fn get_trigger(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> error::JsonResult<Trigger> {
let mut tx = user_db.begin(&authed).await?;
let path = path.to_path();
let trigger = sqlx::query_as!(
Trigger,
r#"SELECT workspace_id, path, route_path, route_path_key, script_path, is_flow, http_method as "http_method: _", edited_by, email, edited_at, extra_perms, is_async, requires_auth
FROM http_trigger
WHERE workspace_id = $1 AND path = $2"#,
w_id,
path,
)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let trigger = not_found_if_none(trigger, "Trigger", path)?;
Ok(Json(trigger))
}
async fn create_trigger(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(ct): Json<NewTrigger>,
) -> error::Result<(StatusCode, String)> {
require_admin(authed.is_admin, &authed.username)?;
let route_path_key = ROUTE_PATH_KEY_RE.replace_all(ct.route_path.as_str(), ":key");
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"INSERT INTO http_trigger (workspace_id, path, route_path, route_path_key, script_path, is_flow, is_async, requires_auth, http_method, edited_by, email, edited_at) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, now())",
w_id,
ct.path,
ct.route_path,
&route_path_key,
ct.script_path,
ct.is_flow,
ct.is_async,
ct.requires_auth,
ct.http_method as HttpMethod,
&authed.username,
&authed.email
)
.execute(&mut *tx).await?;
audit_log(
&mut *tx,
&authed,
"http_triggers.create",
ActionKind::Create,
&w_id,
Some(ct.path.as_str()),
None,
)
.await?;
tx.commit().await?;
Ok((StatusCode::CREATED, format!("{}", ct.path)))
}
async fn update_trigger(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(ct): Json<EditTrigger>,
) -> error::Result<String> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
if authed.is_admin {
if ct.route_path.is_none() {
return Err(error::Error::BadRequest(
"route_path is required".to_string(),
));
}
let route_path_key =
ROUTE_PATH_KEY_RE.replace_all(ct.route_path.as_ref().unwrap().as_str(), ":key");
sqlx::query!(
"UPDATE http_trigger
SET route_path = $1, route_path_key = $2, script_path = $3, path = $4, is_flow = $5, http_method = $6, edited_by = $7, email = $8, is_async = $9, requires_auth = $10, edited_at = now()
WHERE workspace_id = $11 AND path = $12",
ct.route_path,
&route_path_key,
ct.script_path,
ct.path,
ct.is_flow,
ct.http_method as HttpMethod,
&authed.username,
&authed.email,
ct.is_async,
ct.requires_auth,
w_id,
path,
)
.execute(&mut *tx).await?;
} else {
sqlx::query!(
"UPDATE http_trigger SET script_path = $1, path = $2, is_flow = $3, http_method = $4, edited_by = $5, email = $6, is_async = $7, requires_auth = $8, edited_at = now()
WHERE workspace_id = $9 AND path = $10",
ct.script_path,
ct.path,
ct.is_flow,
ct.http_method as HttpMethod,
&authed.username,
&authed.email,
ct.is_async,
ct.requires_auth,
w_id,
path,
)
.execute(&mut *tx).await?;
}
audit_log(
&mut *tx,
&authed,
"http_triggers.update",
ActionKind::Create,
&w_id,
Some(path),
None,
)
.await?;
tx.commit().await?;
Ok(path.to_string())
}
async fn delete_trigger(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> error::Result<String> {
require_admin(authed.is_admin, &authed.username)?;
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"DELETE FROM http_trigger WHERE workspace_id = $1 AND path = $2",
w_id,
path,
)
.execute(&mut *tx)
.await?;
audit_log(
&mut *tx,
&authed,
"http_triggers.delete",
ActionKind::Delete,
&w_id,
Some(path),
None,
)
.await?;
tx.commit().await?;
Ok(format!("HTTP trigger {path} deleted"))
}
async fn used(Extension(db): Extension<DB>, Path(w_id): Path<String>) -> JsonResult<bool> {
let used = sqlx::query_scalar!(
r#"SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1)"#,
w_id,
)
.fetch_one(&db)
.await?
.unwrap_or(false);
Ok(Json(used))
}
async fn exists_trigger(
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<bool> {
let path = path.to_path();
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM http_trigger WHERE path = $1 AND workspace_id = $2)",
path,
w_id,
)
.fetch_one(&db)
.await?
.unwrap_or(false);
Ok(Json(exists))
}
#[derive(Deserialize)]
struct RouteExists {
route_path: String,
http_method: HttpMethod,
}
async fn exists_route(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(RouteExists { route_path, http_method }): Json<RouteExists>,
) -> JsonResult<bool> {
let route_path_key = ROUTE_PATH_KEY_RE.replace_all(route_path.as_str(), ":key");
let exists = if *CLOUD_HOSTED {
sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND workspace_id = $2 AND http_method = $3)",
&route_path_key,
w_id,
http_method as HttpMethod
)
.fetch_one(&db)
.await?
.unwrap_or(false)
} else {
sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM http_trigger WHERE route_path_key = $1 AND http_method = $2)",
&route_path_key,
http_method as HttpMethod
)
.fetch_one(&db)
.await?
.unwrap_or(false)
};
Ok(Json(exists))
}
struct TriggerRoute {
path: String,
script_path: String,
is_flow: bool,
route_path: String,
workspace_id: String,
is_async: bool,
requires_auth: bool,
edited_by: String,
email: String,
http_method: HttpMethod,
}
async fn fetch_api_authed(
username: String,
email: String,
w_id: &str,
db: &DB,
username_override: String,
) -> error::Result<ApiAuthed> {
let permissioned_as = username_to_permissioned_as(username.as_str());
let authed =
fetch_authed_from_permissioned_as(permissioned_as, email.clone(), w_id, db).await?;
Ok(ApiAuthed {
username: username,
email: email,
is_admin: authed.is_admin,
is_operator: authed.is_operator,
groups: authed.groups,
folders: authed.folders,
scopes: authed.scopes,
username_override: Some(username_override),
})
}
async fn get_http_route_trigger(
route_path: &str,
opt_authed: Option<ApiAuthed>,
db: &DB,
user_db: UserDB,
) -> error::Result<(TriggerRoute, String, HashMap<String, String>, ApiAuthed)> {
let (mut triggers, route_path) = if *CLOUD_HOSTED {
let mut splitted = route_path.split("/");
let w_id = splitted.next().ok_or_else(|| {
error::Error::BadRequest("Missing workspace id in route path".to_string())
})?;
let route_path = StripPath(splitted.collect::<Vec<_>>().join("/"));
let triggers = sqlx::query_as!(
TriggerRoute,
r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger WHERE workspace_id = $1"#,
w_id
)
.fetch_all(db)
.await?;
(triggers, route_path)
} else {
let triggers = sqlx::query_as!(
TriggerRoute,
r#"SELECT path, script_path, is_flow, route_path, workspace_id, is_async, requires_auth, edited_by, email, http_method as "http_method: _" FROM http_trigger"#,
)
.fetch_all(db)
.await?;
(triggers, StripPath(route_path.to_string()))
};
let mut router = matchit::Router::new();
for (idx, trigger) in triggers.iter().enumerate() {
let route_path = trigger.route_path.clone();
router.insert(route_path.as_str(), idx).unwrap_or_else(|e| {
tracing::warn!(
"Failed to consider http trigger route {}: {:?}",
route_path,
e,
);
});
}
let trigger_idx = router.at(route_path.0.as_str()).ok();
let matchit::Match { value: trigger_idx, params } =
not_found_if_none(trigger_idx, "Trigger", route_path.0.as_str())?;
let trigger = triggers.remove(trigger_idx.to_owned());
let params: HashMap<String, String> = params
.iter()
.map(|(k, v)| (k.to_string(), v.to_string()))
.collect();
let username_override = if trigger.requires_auth {
if let Some(authed) = opt_authed {
// check that the user has access to the trigger
let mut tx = user_db.begin(&authed).await?;
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM http_trigger WHERE workspace_id = $1 AND path = $2)",
trigger.workspace_id,
trigger.path
)
.fetch_one(&mut *tx)
.await?
.unwrap_or(false);
tx.commit().await?;
if exists {
Some(authed.display_username().to_owned())
} else {
return Err(error::Error::NotAuthorized("Unauthorized".to_string()));
}
} else {
return Err(error::Error::NotAuthorized(
"Requires authentication".to_string(),
));
}
} else {
None
};
let authed = fetch_api_authed(
trigger.edited_by.clone(),
trigger.email.clone(),
&trigger.workspace_id,
&db,
username_override.unwrap_or("anonymous".to_string()),
)
.await?;
Ok((trigger, route_path.0, params, authed))
}
async fn route_job(
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Extension(rsmq): Extension<Option<rsmq_async::MultiplexedRsmq>>,
Path(route_path): Path<StripPath>,
OptAuthed(opt_authed): OptAuthed,
Query(query): Query<HashMap<String, String>>,
method: http::Method,
headers: HeaderMap,
mut args: PushArgsOwned,
) -> impl IntoResponse {
let route_path = route_path.to_path();
let (trigger, called_path, params, authed) =
match get_http_route_trigger(route_path, opt_authed, &db, user_db.clone()).await {
Ok(trigger) => trigger,
Err(e) => return e.into_response(),
};
let headers = headers
.iter()
.map(|(k, v)| (k.to_string(), v.to_str().unwrap_or("").to_string()))
.collect::<HashMap<String, String>>();
let extra = args.extra.get_or_insert_with(HashMap::new);
extra.insert(
"wm_trigger".to_string(),
to_raw_value(&serde_json::json!({
"kind": "http",
"http": {
"route": trigger.route_path,
"path": called_path,
"method": method.to_string().to_lowercase(),
"params": params,
"query": query,
"headers": headers
},
})),
);
let http_method = http::Method::from(trigger.http_method);
if http_method != method {
return error::Error::BadRequest("Invalid HTTP method".to_string()).into_response();
}
let label_prefix = Some(format!(
"http-{}-{}-",
http_method.as_str().to_lowercase(),
trigger.route_path
));
let run_query = RunJobQuery::default();
if trigger.is_flow {
if trigger.is_async {
run_flow_by_path_inner(
authed,
db,
user_db,
rsmq,
trigger.workspace_id.clone(),
StripPath(trigger.script_path.to_owned()),
run_query,
args,
label_prefix,
)
.await
.into_response()
} else {
run_wait_result_flow_by_path_internal(
db,
run_query,
StripPath(trigger.script_path.to_owned()),
authed,
rsmq,
user_db,
args,
trigger.workspace_id.clone(),
label_prefix,
)
.await
.into_response()
}
} else {
if trigger.is_async {
run_script_by_path_inner(
authed,
db,
user_db,
rsmq,
trigger.workspace_id.clone(),
StripPath(trigger.script_path.to_owned()),
run_query,
args,
label_prefix,
)
.await
.into_response()
} else {
run_wait_result_script_by_path_internal(
db,
run_query,
StripPath(trigger.script_path.to_owned()),
authed,
rsmq,
user_db,
trigger.workspace_id.clone(),
args,
label_prefix,
)
.await
.into_response()
}
}
}
backend/windmill-api/src/jobs.rs
+78 -24
View File
@@ -58,7 +58,7 @@ use tower_http::cors::{Any, CorsLayer};
use urlencoding::encode;
use windmill_audit::audit_ee::{audit_log, AuditAuthor};
use windmill_audit::ActionKind;
use windmill_common::worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE, SERVER_CONFIG};
use windmill_common::worker::{to_raw_value, CUSTOM_TAGS_PER_WORKSPACE};
use windmill_common::{
db::UserDB,
error::{self, to_anyhow, Error},
@@ -556,9 +556,10 @@ pub async fn get_path_tag_limits_cache_for_hash<'c, R: rsmq_async::RsmqConnectio
Option<i16>,
Option<bool>,
Option<i32>,
Option<bool>,
)> {
let script = sqlx::query!(
"select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where hash = $1 AND workspace_id = $2",
"select path, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where hash = $1 AND workspace_id = $2",
hash,
w_id
)
@@ -583,6 +584,7 @@ pub async fn get_path_tag_limits_cache_for_hash<'c, R: rsmq_async::RsmqConnectio
script.priority,
script.delete_after_use,
script.timeout,
script.has_preprocessor,
))
}
@@ -1058,6 +1060,7 @@ pub struct RunJobQuery {
pub tag: Option<String>,
pub timeout: Option<i32>,
pub cache_ttl: Option<i32>,
pub skip_preprocessor: Option<bool>,
}
impl RunJobQuery {
@@ -2784,14 +2787,18 @@ pub async fn run_flow_by_path_inner(
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
let (tag, dedicated_worker) = sqlx::query!(
"SELECT tag, dedicated_worker from flow WHERE path = $1 and workspace_id = $2",
let (tag, dedicated_worker, has_preprocessor) = sqlx::query!(
"SELECT tag, dedicated_worker, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor
FROM flow
LEFT JOIN flow_version
ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
WHERE flow.path = $1 and flow.workspace_id = $2",
flow_path,
w_id
)
.fetch_optional(&mut tx)
.await?
.map(|x| (x.tag, x.dedicated_worker))
.map(|x| (x.tag, x.dedicated_worker, x.has_preprocessor))
.ok_or_else(|| {
Error::NotFound(format!(
"flow not found at path {flow_path} in workspace {w_id}"
@@ -2807,7 +2814,12 @@ pub async fn run_flow_by_path_inner(
&db,
tx,
&w_id,
JobPayload::Flow { path: flow_path.to_string(), dedicated_worker },
JobPayload::Flow {
path: flow_path.to_string(),
dedicated_worker,
apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
&& has_preprocessor.unwrap_or(false),
},
PushArgs { args: &args.args, extra: args.extra },
&label_prefix
.map(|x| x + authed.display_username())
@@ -2973,7 +2985,7 @@ pub async fn run_script_by_path_inner(
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
let (job_payload, tag, _delete_after_use, timeout) =
script_path_to_payload(script_path, &mut tx, &w_id).await?;
script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?;
let scheduled_for = run_query.get_scheduled_for(&db).await?;
let tag = run_query.tag.clone().or(tag);
@@ -3049,7 +3061,15 @@ pub async fn run_workflow_as_code(
None,
run_query.timeout,
),
JobKind::Script => script_path_to_payload(job.script_path(), &mut tx, &w_id).await?,
JobKind::Script => {
script_path_to_payload(
job.script_path(),
&mut tx,
&w_id,
run_query.skip_preprocessor,
)
.await?
}
_ => return Err(anyhow::anyhow!("Not supported").into()),
};
@@ -3145,6 +3165,17 @@ impl Drop for Guard {
}
}
use std::sync::Arc;
use tokio::sync::RwLock;
lazy_static::lazy_static! {
pub static ref TIMEOUT_WAIT_RESULT: Arc<RwLock<Option<u64>>> = Arc::new(RwLock::new(
std::env::var("TIMEOUT_WAIT_RESULT")
.ok()
.and_then(|x| x.parse::<u64>().ok())
));
}
#[derive(Deserialize)]
pub struct WindmillCompositeResult {
windmill_status_code: Option<u16>,
@@ -3159,7 +3190,7 @@ async fn run_wait_result(
username: &str,
) -> error::Result<Response> {
let mut result = None;
let timeout = SERVER_CONFIG.read().await.timeout_wait_result.clone();
let timeout = TIMEOUT_WAIT_RESULT.read().await.clone().unwrap_or(600);
let timeout_ms = if timeout <= 0 {
2000
} else {
@@ -3451,7 +3482,7 @@ pub async fn run_wait_result_job_by_path_get(
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
let (job_payload, tag, delete_after_use, timeout) =
script_path_to_payload(script_path, &mut tx, &w_id).await?;
script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?;
let tag = run_query.tag.clone().or(tag);
check_tag_available_for_workspace(&w_id, &tag).await?;
@@ -3527,7 +3558,7 @@ pub async fn run_wait_result_flow_by_path_get(
let args = PushArgsOwned { extra: Some(payload_args), args: HashMap::new() };
run_wait_result_flow_by_path_internal(
db, run_query, flow_path, authed, rsmq, user_db, args, w_id,
db, run_query, flow_path, authed, rsmq, user_db, args, w_id, None,
)
.await
}
@@ -3553,11 +3584,12 @@ pub async fn run_wait_result_script_by_path(
user_db,
w_id,
args,
None,
)
.await
}
async fn run_wait_result_script_by_path_internal(
pub async fn run_wait_result_script_by_path_internal(
db: sqlx::Pool<Postgres>,
run_query: RunJobQuery,
script_path: StripPath,
@@ -3566,6 +3598,7 @@ async fn run_wait_result_script_by_path_internal(
user_db: UserDB,
w_id: String,
args: PushArgsOwned,
label_prefix: Option<String>,
) -> error::Result<Response> {
check_queue_too_long(&db, QUEUE_LIMIT_WAIT_RESULT.or(run_query.queue_limit)).await?;
let script_path = script_path.to_path();
@@ -3574,7 +3607,7 @@ async fn run_wait_result_script_by_path_internal(
let mut tx: QueueTransaction<'_, _> = (rsmq, user_db.begin(&authed).await?).into();
let (job_payload, tag, delete_after_use, timeout) =
script_path_to_payload(script_path, &mut tx, &w_id).await?;
script_path_to_payload(script_path, &mut tx, &w_id, run_query.skip_preprocessor).await?;
let tag = run_query.tag.clone().or(tag);
check_tag_available_for_workspace(&w_id, &tag).await?;
@@ -3587,7 +3620,9 @@ async fn run_wait_result_script_by_path_internal(
&w_id,
job_payload,
PushArgs { args: &args.args, extra: args.extra },
authed.display_username(),
&label_prefix
.map(|x| x + authed.display_username())
.unwrap_or_else(|| authed.display_username().to_string()),
&authed.email,
username_to_permissioned_as(&authed.username),
None,
@@ -3644,6 +3679,7 @@ pub async fn run_wait_result_script_by_hash(
priority,
delete_after_use,
timeout,
has_preprocessor,
) = get_path_tag_limits_cache_for_hash(&mut tx, &w_id, hash).await?;
if let Some(run_query_cache_ttl) = run_query.cache_ttl {
cache_ttl = Some(run_query_cache_ttl);
@@ -3669,6 +3705,8 @@ pub async fn run_wait_result_script_by_hash(
language,
dedicated_worker,
priority,
apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
&& has_preprocessor.unwrap_or(false),
},
PushArgs { args: &args.args, extra: args.extra },
authed.display_username(),
@@ -3712,12 +3750,12 @@ pub async fn run_wait_result_flow_by_path(
check_license_key_valid().await?;
run_wait_result_flow_by_path_internal(
db, run_query, flow_path, authed, rsmq, user_db, args, w_id,
db, run_query, flow_path, authed, rsmq, user_db, args, w_id, None,
)
.await
}
async fn run_wait_result_flow_by_path_internal(
pub async fn run_wait_result_flow_by_path_internal(
db: sqlx::Pool<Postgres>,
run_query: RunJobQuery,
flow_path: StripPath,
@@ -3726,6 +3764,7 @@ async fn run_wait_result_flow_by_path_internal(
user_db: UserDB,
args: PushArgsOwned,
w_id: String,
label_prefix: Option<String>,
) -> error::Result<Response> {
check_queue_too_long(&db, run_query.queue_limit).await?;
@@ -3736,8 +3775,8 @@ async fn run_wait_result_flow_by_path_internal(
let scheduled_for = run_query.get_scheduled_for(&db).await?;
let (tag, dedicated_worker, early_return) = sqlx::query!(
"SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return
let (tag, dedicated_worker, early_return, has_preprocessor) = sqlx::query!(
"SELECT tag, dedicated_worker, flow_version.value->>'early_return' as early_return, flow_version.value->>'preprocessor_module' IS NOT NULL as has_preprocessor
FROM flow
LEFT JOIN flow_version
ON flow_version.id = flow.versions[array_upper(flow.versions, 1)]
@@ -3747,7 +3786,7 @@ async fn run_wait_result_flow_by_path_internal(
)
.fetch_optional(&mut tx)
.await?
.map(|x| (x.tag, x.dedicated_worker, x.early_return))
.map(|x| (x.tag, x.dedicated_worker, x.early_return, x.has_preprocessor))
.ok_or_else(|| {
Error::NotFound(format!(
"flow not found at path {flow_path} in workspace {w_id}"
@@ -3763,9 +3802,16 @@ async fn run_wait_result_flow_by_path_internal(
&db,
tx,
&w_id,
JobPayload::Flow { path: flow_path.to_string(), dedicated_worker },
JobPayload::Flow {
path: flow_path.to_string(),
dedicated_worker,
apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
&& has_preprocessor.unwrap_or(false),
},
PushArgs { args: &args.args, extra: args.extra },
authed.display_username(),
&label_prefix
.map(|x| x + authed.display_username())
.unwrap_or_else(|| authed.display_username().to_string()),
&authed.email,
username_to_permissioned_as(&authed.username),
scheduled_for,
@@ -4202,6 +4248,7 @@ async fn add_batch_jobs(
_priority,
_delete_after_use,
timeout,
_,
) = get_latest_deployed_hash_for_path(&db, &w_id, &path).await?;
(
Some(script_hash),
@@ -4228,7 +4275,11 @@ async fn add_batch_jobs(
JobPayload::RawFlow { value: fv.clone(), path: None, restarted_from: None }
} else {
if let Some(path) = batch_info.path.as_ref() {
JobPayload::Flow { path: path.to_string(), dedicated_worker: None }
JobPayload::Flow {
path: path.to_string(),
dedicated_worker: None,
apply_preprocessor: false,
}
} else {
Err(anyhow::anyhow!(
"Path is required if no value is not provided"
@@ -4450,6 +4501,7 @@ pub async fn run_job_by_hash_inner(
priority,
_delete_after_use, // not taken into account in async endpoints
timeout,
has_preprocessor,
) = get_path_tag_limits_cache_for_hash(&mut tx, &w_id, hash).await?;
check_scopes(&authed, || format!("run:script/{path}"))?;
if let Some(run_query_cache_ttl) = run_query.cache_ttl {
@@ -4475,6 +4527,8 @@ pub async fn run_job_by_hash_inner(
language,
dedicated_worker,
priority,
apply_preprocessor: !run_query.skip_preprocessor.unwrap_or(false)
&& has_preprocessor.unwrap_or(false),
},
PushArgs { args: &args.args, extra: args.extra },
&label_prefix
@@ -4609,8 +4663,8 @@ async fn get_job_update(
"progress_perc"
)
.fetch_one(&db)
.await?
.fetch_optional(&db)
.await?.and_then(|inner| inner)
} else {
None
};
backend/windmill-api/src/lib.rs
+8 -2
View File
@@ -60,6 +60,7 @@ mod flows;
mod folders;
mod granular_acls;
mod groups;
mod http_triggers;
mod indexer_ee;
mod inputs;
mod integration;
@@ -283,7 +284,8 @@ pub async fn run_server(
)
.nest("/variables", variables::workspaced_service())
.nest("/workspaces", workspaces::workspaced_service())
.nest("/oidc", oidc_ee::workspaced_service()),
.nest("/oidc", oidc_ee::workspaced_service())
.nest("/http_triggers", http_triggers::workspaced_service()),
)
.nest("/workspaces", workspaces::global_service())
.nest(
@@ -336,7 +338,7 @@ pub async fn run_server(
)
.nest(
"/w/:workspace_id/capture_u",
capture::global_service().layer(cors),
capture::global_service().layer(cors.clone()),
)
.nest(
"/auth",
@@ -346,6 +348,10 @@ pub async fn run_server(
"/oauth",
oauth2_ee::global_service().layer(Extension(Arc::clone(&sp_extension))),
)
.nest(
"/r",
http_triggers::routes_global_service().layer(from_extractor::<OptAuthed>()),
)
.route("/version", get(git_v))
.route("/uptodate", get(is_up_to_date))
.route("/ee_license", get(ee_license))
backend/windmill-api/src/scripts.rs
+9 -4
View File
@@ -103,6 +103,8 @@ pub struct ScriptWDraft {
pub visible_to_runner_only: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub no_main_func: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub has_preprocessor: Option<bool>,
}
pub fn global_service() -> Router {
@@ -581,6 +583,7 @@ async fn create_script_internal<'c>(
|| ns.language == ScriptLang::Bunnative
|| ns.language == ScriptLang::Deno
|| ns.language == ScriptLang::Rust
|| ns.language == ScriptLang::Ansible
|| ns.language == ScriptLang::Php)
{
Some(String::new())
@@ -612,8 +615,8 @@ async fn create_script_internal<'c>(
content, created_by, schema, is_template, extra_perms, lock, language, kind, tag, \
draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, \
dedicated_worker, ws_error_handler_muted, priority, restart_unless_cancelled, \
delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30)",
delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, codebase, has_preprocessor) \
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23, $24, $25, $26, $27, $28, $29, $30, $31)",
&w_id,
&hash.0,
ns.path,
@@ -643,7 +646,8 @@ async fn create_script_internal<'c>(
ns.concurrency_key,
ns.visible_to_runner_only,
ns.no_main_func,
codebase
codebase,
ns.has_preprocessor,
)
.execute(&mut tx)
.await?;
@@ -738,6 +742,7 @@ async fn create_script_internal<'c>(
},
);
}
let permissioned_as = username_to_permissioned_as(&authed.username);
if needs_lock_gen {
let tag = if ns.dedicated_worker.is_some_and(|x| x) {
@@ -878,7 +883,7 @@ async fn get_script_by_path_w_draft(
let mut tx = user_db.begin(&authed).await?;
let script_o = sqlx::query_as::<_, ScriptWDraft>(
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func FROM script LEFT JOIN draft ON
"SELECT hash, script.path, summary, description, content, language, kind, tag, schema, draft_only, envs, concurrent_limit, concurrency_time_window_s, cache_ttl, ws_error_handler_muted, draft.value as draft, dedicated_worker, priority, restart_unless_cancelled, delete_after_use, timeout, concurrency_key, visible_to_runner_only, no_main_func, has_preprocessor FROM script LEFT JOIN draft ON
script.path = draft.path AND script.workspace_id = draft.workspace_id AND draft.typ = 'script'
WHERE script.path = $1 AND script.workspace_id = $2 \
AND script.created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND \
backend/windmill-api/src/settings.rs
+28 -1
View File
@@ -24,6 +24,8 @@ use axum::{
#[cfg(feature = "enterprise")]
use axum::extract::Query;
#[cfg(feature = "enterprise")]
use windmill_common::ee::{send_critical_alert, CriticalAlertKind, CriticalErrorChannel};
use serde::Deserialize;
use windmill_common::{
error::{self, JsonResult, Result},
@@ -55,7 +57,8 @@ pub fn global_service() -> Router {
get(get_latest_key_renewal_attempt),
)
.route("/renew_license_key", post(renew_license_key))
.route("/customer_portal", post(create_customer_portal_session));
.route("/customer_portal", post(create_customer_portal_session))
.route("/test_critical_channels", post(test_critical_channels));
#[cfg(feature = "parquet")]
{
@@ -387,3 +390,27 @@ pub async fn create_customer_portal_session(
return Ok(url);
}
#[cfg(feature = "enterprise")]
pub async fn test_critical_channels(
Extension(db): Extension<DB>,
authed: ApiAuthed,
Json(test_critical_channels): Json<Vec<CriticalErrorChannel>>,
) -> Result<String> {
require_super_admin(&db, &authed.email).await?;
#[cfg(feature = "enterprise")]
send_critical_alert(
"Test critical error".to_string(),
&db,
CriticalAlertKind::CriticalError,
Some(test_critical_channels),
)
.await;
Ok("Sent test critical error".to_string())
}
#[cfg(not(feature = "enterprise"))]
pub async fn test_critical_channels() -> Result<String> {
Ok("Critical channels require EE".to_string())
}
backend/windmill-api/src/users.rs
+2 -2
View File
@@ -51,7 +51,7 @@ use windmill_audit::ActionKind;
use windmill_common::global_settings::AUTOMATE_USERNAME_CREATION_SETTING;
use windmill_common::users::truncate_token;
use windmill_common::utils::{paginate, send_email};
use windmill_common::worker::{CLOUD_HOSTED, SERVER_CONFIG};
use windmill_common::worker::{CLOUD_HOSTED, SMTP_CONFIG};
use windmill_common::{
auth::{get_folders_for_user, get_groups_for_user, JWTAuthClaims, JWT_SECRET},
db::UserDB,
@@ -1966,7 +1966,7 @@ pub fn send_email_if_possible(subject: &str, content: &str, to: &str) {
}
pub async fn send_email_if_possible_intern(subject: &str, content: &str, to: String) -> Result<()> {
if let Some(smtp) = SERVER_CONFIG.read().await.smtp.clone() {
if let Some(smtp) = SMTP_CONFIG.read().await.clone() {
send_email(subject, content, vec![to], smtp, None).await?;
}
return Ok(());
backend/windmill-api/src/workspaces.rs
+3
View File
@@ -2317,6 +2317,8 @@ struct ScriptMetadata {
pub codebase: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub concurrency_key: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub has_preprocessor: Option<bool>,
}
pub fn is_none_or_false(val: &Option<bool>) -> bool {
@@ -2602,6 +2604,7 @@ async fn tarball_workspace(
no_main_func: script.no_main_func,
codebase: script.codebase,
concurrency_key: script.concurrency_key,
has_preprocessor: script.has_preprocessor,
};
let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
archive
backend/windmill-common/src/ee.rs
+7 -1
View File
@@ -36,7 +36,13 @@ pub enum CriticalAlertKind {
}
#[cfg(feature = "enterprise")]
pub async fn send_critical_alert(_error_message: String, _db: &DB, _kind: CriticalAlertKind) {}
pub async fn send_critical_alert(
_error_message: String,
_db: &DB,
_kind: CriticalAlertKind,
_channels: Option<Vec<CriticalErrorChannel>>,
) {
}
#[cfg(feature = "enterprise")]
pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
backend/windmill-common/src/flow_status.rs
+13 -1
View File
@@ -29,6 +29,7 @@ pub struct FlowStatus {
pub step: i32,
pub modules: Vec<FlowStatusModule>,
pub failure_module: Box<FlowStatusModuleWParent>,
pub preprocessor_module: Option<FlowStatusModule>,
#[serde(skip_serializing_if = "HashMap::is_empty")]
#[serde(default)]
@@ -355,7 +356,11 @@ impl FlowStatusModule {
impl FlowStatus {
pub fn new(f: &FlowValue) -> Self {
Self {
step: 0,
step: if f.preprocessor_module.is_some() {
-1
} else {
0
},
approval_conditions: None,
modules: f
.modules
@@ -372,6 +377,13 @@ impl FlowStatus {
.unwrap_or_else(|| "failure".to_string()),
},
}),
preprocessor_module: if f.preprocessor_module.is_some() {
Some(FlowStatusModule::WaitingForPriorSteps {
id: f.preprocessor_module.as_ref().unwrap().id.clone(),
})
} else {
None
},
cleanup_module: FlowCleanupModule { flow_jobs_to_clean: vec![] },
retry: RetryStatus { fail_count: 0, failed_jobs: vec![] },
restarted_from: None,
backend/windmill-common/src/flows.rs
+2
View File
@@ -102,6 +102,8 @@ pub struct FlowValue {
#[serde(default)]
pub failure_module: Option<Box<FlowModule>>,
#[serde(default)]
pub preprocessor_module: Option<Box<FlowModule>>,
#[serde(default)]
#[serde(skip_serializing_if = "is_default")]
pub same_worker: bool,
backend/windmill-common/src/global_settings.rs
+2
View File
@@ -14,6 +14,8 @@ pub const EXTRA_PIP_INDEX_URL_SETTING: &str = "pip_extra_index_url";
pub const PIP_INDEX_URL_SETTING: &str = "pip_index_url";
pub const SCIM_TOKEN_SETTING: &str = "scim_token";
pub const SAML_METADATA_SETTING: &str = "saml_metadata";
pub const SMTP_SETTING: &str = "smtp_settings";
pub const TIMEOUT_WAIT_RESULT_SETTING: &str = "timeout_wait_result";
pub const UNIQUE_ID_SETTING: &str = "uid";
pub const DISABLE_STATS_SETTING: &str = "disable_stats";
backend/windmill-common/src/jobs.rs
+15 -2
View File
@@ -11,6 +11,8 @@ use uuid::Uuid;
pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE";
pub const PREPROCESSOR_FAKE_ENTRYPOINT: &str = "__WM_PREPROCESSOR";
use crate::{
error::{self, to_anyhow, Error},
flow_status::{FlowStatus, RestartedFrom},
@@ -302,6 +304,7 @@ pub enum JobPayload {
dedicated_worker: Option<bool>,
language: ScriptLang,
priority: Option<i16>,
apply_preprocessor: bool,
},
Code(RawCode),
Dependencies {
@@ -331,6 +334,7 @@ pub enum JobPayload {
Flow {
path: String,
dedicated_worker: Option<bool>,
apply_preprocessor: bool,
},
RestartedFlow {
completed_job_id: Uuid,
@@ -383,6 +387,7 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre
script_path: &str,
db: E,
w_id: &str,
skip_preprocessor: Option<bool>,
) -> error::Result<(JobPayload, Option<Tag>, Option<bool>, Option<i32>)> {
let (job_payload, tag, delete_after_use, script_timeout) = if script_path.starts_with("hub/") {
(
@@ -404,6 +409,7 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre
priority,
delete_after_use,
script_timeout,
has_preprocessor,
) = get_latest_deployed_hash_for_path(db, w_id, script_path).await?;
(
JobPayload::ScriptHash {
@@ -416,6 +422,8 @@ pub async fn script_path_to_payload<'e, E: sqlx::Executor<'e, Database = Postgre
language,
dedicated_worker,
priority,
apply_preprocessor: !skip_preprocessor.unwrap_or(false)
&& has_preprocessor.unwrap_or(false),
},
tag,
delete_after_use,
@@ -473,7 +481,7 @@ pub async fn get_payload_tag_from_prefixed_path<'e, E: sqlx::Executor<'e, Databa
w_id: &str,
) -> Result<(JobPayload, Option<String>), Error> {
let (payload, tag, _, _) = if path.starts_with("script/") {
script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id).await?
script_path_to_payload(path.strip_prefix("script/").unwrap(), db, w_id, Some(true)).await?
} else if path.starts_with("flow/") {
let path = path.strip_prefix("flow/").unwrap().to_string();
let r = sqlx::query!(
@@ -486,7 +494,12 @@ pub async fn get_payload_tag_from_prefixed_path<'e, E: sqlx::Executor<'e, Databa
let (tag, dedicated_worker) = r
.map(|x| (x.tag, x.dedicated_worker))
.unwrap_or_else(|| (None, None));
(JobPayload::Flow { path, dedicated_worker }, tag, None, None)
(
JobPayload::Flow { path, dedicated_worker, apply_preprocessor: false },
tag,
None,
None,
)
} else {
return Err(Error::BadRequest(format!(
"path must start with script/ or flow/ (got {})",
backend/windmill-common/src/lib.rs
+3 -1
View File
@@ -271,9 +271,10 @@ pub async fn get_latest_deployed_hash_for_path<'e, E: sqlx::Executor<'e, Databas
Option<i16>,
Option<bool>,
Option<i32>,
Option<bool>,
)> {
let r_o = sqlx::query!(
"select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout from script where path = $1 AND workspace_id = $2 AND
"select hash, tag, concurrency_key, concurrent_limit, concurrency_time_window_s, cache_ttl, language as \"language: ScriptLang\", dedicated_worker, priority, delete_after_use, timeout, has_preprocessor from script where path = $1 AND workspace_id = $2 AND
created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND workspace_id = $2 AND
deleted = false AND lock IS not NULL AND lock_error_logs IS NULL)",
script_path,
@@ -296,6 +297,7 @@ pub async fn get_latest_deployed_hash_for_path<'e, E: sqlx::Executor<'e, Databas
script.priority,
script.delete_after_use,
script.timeout,
script.has_preprocessor,
))
}
backend/windmill-common/src/scripts.rs
+3
View File
@@ -193,6 +193,8 @@ pub struct Script {
pub no_main_func: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub codebase: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub has_preprocessor: Option<bool>,
}
#[derive(Serialize, sqlx::FromRow)]
@@ -289,6 +291,7 @@ pub struct NewScript {
pub visible_to_runner_only: Option<bool>,
pub no_main_func: Option<bool>,
pub codebase: Option<String>,
pub has_preprocessor: Option<bool>,
}
fn lock_deserialize<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
backend/windmill-common/src/server.rs
+6 -25
View File
@@ -13,22 +13,20 @@ pub struct Smtp {
}
#[derive(Serialize, Deserialize, PartialEq)]
pub struct ServerConfigOpt {
pub struct SmtpConfigOpt {
pub smtp_host: Option<String>,
pub smtp_username: Option<String>,
pub smtp_password: Option<String>,
pub smtp_port: Option<u16>,
pub smtp_from: Option<String>,
pub smtp_tls_implicit: Option<bool>,
pub timeout_wait_result: Option<u64>,
}
pub async fn load_server_config(db: &DB) -> error::Result<ServerConfig> {
let config: ServerConfigOpt =
sqlx::query_scalar!("SELECT config FROM config WHERE name = 'server'",)
pub async fn load_smtp_config(db: &DB) -> error::Result<Option<Smtp>> {
let config: SmtpConfigOpt =
sqlx::query_scalar!("SELECT value FROM global_settings WHERE name = 'smtp_settings'",)
.fetch_optional(db)
.await?
.flatten()
.map(|x| serde_json::from_value(x).ok())
.flatten()
.unwrap_or_default();
@@ -77,20 +75,10 @@ pub async fn load_server_config(db: &DB) -> error::Result<ServerConfig> {
tracing::warn!("SMTP not configured");
}
Ok(ServerConfig {
smtp,
timeout_wait_result: config
.timeout_wait_result
.ok_or(
std::env::var("TIMEOUT_WAIT_RESULT")
.ok()
.and_then(|x| x.parse::<u64>().ok()),
)
.unwrap_or(600),
})
Ok(smtp)
}
impl Default for ServerConfigOpt {
impl Default for SmtpConfigOpt {
fn default() -> Self {
Self {
smtp_from: None,
@@ -99,13 +87,6 @@ impl Default for ServerConfigOpt {
smtp_port: None,
smtp_tls_implicit: None,
smtp_username: None,
timeout_wait_result: Default::default(),
}
}
}
#[derive(PartialEq, Clone, Debug)]
pub struct ServerConfig {
pub smtp: Option<Smtp>,
pub timeout_wait_result: u64,
}
backend/windmill-common/src/utils.rs
+8 -2
View File
@@ -288,7 +288,7 @@ pub async fn report_critical_error(error_message: String, _db: DB) -> () {
}
#[cfg(feature = "enterprise")]
send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError).await;
send_critical_alert(error_message, &_db, CriticalAlertKind::CriticalError, None).await;
}
pub async fn report_recovered_critical_error(message: String, _db: DB) -> () {
@@ -304,5 +304,11 @@ pub async fn report_recovered_critical_error(message: String, _db: DB) -> () {
tracing::error!("Failed to save critical error to database: {}", err);
}
#[cfg(feature = "enterprise")]
send_critical_alert(message, &_db, CriticalAlertKind::RecoveredCriticalError).await;
send_critical_alert(
message,
&_db,
CriticalAlertKind::RecoveredCriticalError,
None,
)
.await;
}
backend/windmill-common/src/worker.rs
+11 -6
View File
@@ -14,7 +14,7 @@ use std::{
};
use tokio::sync::RwLock;
use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::ServerConfig, DB};
use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::Smtp, DB};
lazy_static::lazy_static! {
pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
@@ -66,8 +66,7 @@ lazy_static::lazy_static! {
pub static ref WORKER_SUSPENDED_PULL_QUERY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
pub static ref SERVER_CONFIG: Arc<RwLock<ServerConfig>> = Arc::new(RwLock::new(ServerConfig { smtp: Default::default(), timeout_wait_result: 20 }));
pub static ref SMTP_CONFIG: Arc<RwLock<Option<Smtp>>> = Arc::new(RwLock::new(None));
pub static ref CLOUD_HOSTED: bool = std::env::var("CLOUD_HOSTED").is_ok();
@@ -195,7 +194,11 @@ fn normalize_path(path: &Path) -> PathBuf {
}
ret
}
pub fn write_file_at_user_defined_location(job_dir: &str, user_defined_path: &str, content: &str) -> error::Result<File> {
pub fn write_file_at_user_defined_location(
job_dir: &str,
user_defined_path: &str,
content: &str,
) -> error::Result<PathBuf> {
let job_dir = Path::new(job_dir);
let user_path = PathBuf::from(user_defined_path);
@@ -210,9 +213,11 @@ pub fn write_file_at_user_defined_location(job_dir: &str, user_defined_path: &st
return Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"Path is outside the allowed job directory.",
).into());
)
.into());
}
let full_path = normalized_full_path.as_path();
if let Some(parent_dir) = full_path.parent() {
std::fs::create_dir_all(parent_dir)?;
}
@@ -220,7 +225,7 @@ pub fn write_file_at_user_defined_location(job_dir: &str, user_defined_path: &st
let mut file = File::create(full_path)?;
file.write_all(content.as_bytes())?;
file.flush()?;
Ok(file)
Ok(normalized_full_path)
}
pub async fn reload_custom_tags_setting(db: &DB) -> error::Result<()> {
backend/windmill-queue/src/jobs.rs
+67 -19
View File
@@ -52,6 +52,7 @@ use windmill_common::{
},
jobs::{
get_payload_tag_from_prefixed_path, CompletedJob, JobKind, JobPayload, QueuedJob, RawCode,
ENTRYPOINT_OVERRIDE, PREPROCESSOR_FAKE_ENTRYPOINT,
},
schedule::Schedule,
scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
@@ -1021,6 +1022,7 @@ pub async fn add_completed_job<
.clone()
.unwrap_or_else(|| ScriptLang::Deno),
priority: queued_job.priority,
apply_preprocessor: false,
},
queued_job
.args
@@ -2864,10 +2866,21 @@ where
(content_type, extra, raw, wrap_body)
};
if content_type.is_none() || content_type.unwrap().starts_with("application/json") {
let no_content_type = content_type.is_none();
if no_content_type || content_type.unwrap().starts_with("application/json") {
let bytes = Bytes::from_request(req, _state)
.await
.map_err(IntoResponse::into_response)?;
if no_content_type && bytes.is_empty() {
if use_raw {
extra.insert("raw_string".to_string(), to_raw_value(&"".to_string()));
}
let mut args = HashMap::new();
if wrap_body {
args.insert("body".to_string(), to_raw_value(&serde_json::json!({})));
}
return Ok(PushArgsOwned { extra: Some(extra), args: args });
}
let str = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
@@ -2992,7 +3005,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
mut tx: PushIsolationLevel<'c, R>,
workspace_id: &str,
job_payload: JobPayload,
args: PushArgs<'d>,
mut args: PushArgs<'d>,
user: &str,
mut email: &str,
mut permissioned_as: String,
@@ -3211,21 +3224,38 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
language,
dedicated_worker,
priority,
} => (
Some(hash.0),
Some(path),
None,
JobKind::Script,
None,
None,
Some(language),
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
cache_ttl,
dedicated_worker,
priority,
),
apply_preprocessor,
} => {
let extra = args.extra.get_or_insert_with(HashMap::new);
if apply_preprocessor {
extra.insert(
ENTRYPOINT_OVERRIDE.to_string(),
to_raw_value(&PREPROCESSOR_FAKE_ENTRYPOINT),
);
extra.entry("wm_trigger".to_string()).or_insert_with(|| {
to_raw_value(&serde_json::json!({
"kind": "webhook",
}))
});
} else {
extra.remove("wm_trigger");
}
(
Some(hash.0),
Some(path),
None,
JobKind::Script,
None,
None,
Some(language),
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
cache_ttl,
dedicated_worker,
priority,
)
}
JobPayload::ScriptHub { path } => {
if path == "hub/7771/slack" || path == "hub/7836/slack" {
permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string();
@@ -3404,9 +3434,13 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
branch_or_iteration_n: restarted_from_val.branch_or_iteration_n,
}),
user_states,
preprocessor_module: None,
}
}
_ => FlowStatus::new(&value), // this is a new flow being pushed, flow_status is set to flow_value
_ => {
value.preprocessor_module = None;
FlowStatus::new(&value)
} // this is a new flow being pushed, flow_status is set to flow_value
};
(
None,
@@ -3473,6 +3507,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
early_return: None,
concurrency_key: custom_concurrency_key.clone(),
priority: priority,
preprocessor_module: None,
};
(
None,
@@ -3490,7 +3525,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
priority,
)
}
JobPayload::Flow { path, dedicated_worker } => {
JobPayload::Flow { path, dedicated_worker, apply_preprocessor } => {
let value_json = fetch_scalar_isolated!(
sqlx::query_as::<_, FlowRawValue>(
"SELECT flow_version.value FROM flow
@@ -3518,6 +3553,18 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
let custom_concurrency_key = value.concurrency_key.clone();
let concurrency_time_window_s = value.concurrency_time_window_s.clone();
let concurrent_limit = value.concurrent_limit.clone();
let extra = args.extra.get_or_insert_with(HashMap::new);
if !apply_preprocessor {
value.preprocessor_module = None;
extra.remove("wm_trigger");
} else {
extra.entry("wm_trigger".to_string()).or_insert_with(|| {
to_raw_value(&serde_json::json!({
"kind": "webhook",
}))
});
}
let status = Some(FlowStatus::new(&value));
(
None,
@@ -3574,6 +3621,7 @@ pub async fn push<'c, 'd, R: rsmq_async::RsmqConnection + Send + 'c>(
branch_or_iteration_n,
}),
user_states,
preprocessor_module: None,
};
(
None,
backend/windmill-queue/src/schedule.rs
+6 -1
View File
@@ -120,7 +120,11 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
.map(|x| (x.tag, x.dedicated_worker))
.unwrap_or_else(|| (None, None));
(
JobPayload::Flow { path: schedule.script_path.clone(), dedicated_worker },
JobPayload::Flow {
path: schedule.script_path.clone(),
dedicated_worker,
apply_preprocessor: false,
},
tag,
None,
)
@@ -188,6 +192,7 @@ pub async fn push_scheduled_job<'c, R: rsmq_async::RsmqConnection + Send + 'c>(
dedicated_worker,
language,
priority,
apply_preprocessor: false,
},
if schedule.tag.as_ref().is_some_and(|x| x != "") {
schedule.tag.clone()
backend/windmill-worker/nsjail/run.ansible.config.proto
+140
View File
@@ -0,0 +1,140 @@
name: "ansible run script"
mode: ONCE
hostname: "ansible"
log_level: ERROR
rlimit_as: 4096
rlimit_cpu: 1000
rlimit_fsize: 1000
rlimit_nofile: 10000
cwd: "/tmp"
clone_newnet: false
clone_newuser: {CLONE_NEWUSER}
keep_caps: false
keep_env: true
mount {
src: "/bin"
dst: "/bin"
is_bind: true
}
mount {
src: "/lib"
dst: "/lib"
is_bind: true
}
mount {
src: "/lib64"
dst: "/lib64"
is_bind: true
mandatory: false
}
mount {
src: "/usr"
dst: "/usr"
is_bind: true
}
mount {
src: "/dev/null"
dst: "/dev/null"
is_bind: true
rw: true
}
mount {
dst: "/dev/shm"
fstype: "tmpfs"
rw: true
is_bind: false
}
mount {
dst: "/tmp"
fstype: "tmpfs"
rw: true
options: "size=800000000"
}
mount {
src: "{JOB_DIR}/main.yml"
dst: "/tmp/main.yml"
is_bind: true
}
mount {
src: "{JOB_DIR}/wrapper.sh"
dst: "/tmp/wrapper.sh"
is_bind: true
}
mount {
src: "{JOB_DIR}/requirements.yml"
dst: "/tmp/requirements.yml"
is_bind: true
mandatory: false
}
mount {
src: "{JOB_DIR}/ansible.cfg"
dst: "/tmp/ansible.cfg"
is_bind: true
}
mount {
src: "{JOB_DIR}/ansible_collections/"
dst: "/tmp/ansible_collections/"
is_bind: true
}
mount {
src: "{JOB_DIR}/args.json"
dst: "/tmp/args.json"
is_bind: true
}
mount {
src: "{JOB_DIR}/result.json"
dst: "/tmp/result_nsjail_mount.json"
rw: true
is_bind: true
}
mount {
src: "/etc"
dst: "/etc"
is_bind: true
}
mount {
src: "/dev/random"
dst: "/dev/random"
is_bind: true
}
mount {
src: "/dev/urandom"
dst: "/dev/urandom"
is_bind: true
}
{SHARED_MOUNT}
{SHARED_DEPENDENCIES}
{FILE_RESOURCES}
iface_no_lo: true
envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH"
envar: "PYTHONPATH={ADDITIONAL_PYTHON_PATHS}"
envar: "HOME=/tmp"
envar: "ANSIBLE_CONFIG=/tmp/ansible.cfg"
backend/windmill-worker/src/ansible_executor.rs
+159 -24
View File
@@ -1,6 +1,9 @@
use std::{collections::HashMap, process::Stdio};
use std::{
collections::HashMap, os::unix::fs::PermissionsExt, path::{Path, PathBuf}, process::Stdio
};
use anyhow::anyhow;
use itertools::Itertools;
use serde_json::value::RawValue;
use tokio::process::Command;
use uuid::Uuid;
@@ -13,12 +16,10 @@ use windmill_parser_yaml::AnsibleRequirements;
use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
get_reserved_variables, handle_child, read_and_check_result,
start_child_process, transform_json,
},
python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile},
AuthedClientBackgroundTask, DISABLE_NSJAIL, HOME_ENV, PATH_ENV, TZ_ENV,
bash_executor::BIN_BASH, common::{
get_reserved_variables, handle_child, read_and_check_result, start_child_process,
transform_json,
}, python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV
};
lazy_static::lazy_static! {
@@ -29,6 +30,8 @@ lazy_static::lazy_static! {
std::env::var("ANSIBLE_GALAXY_PATH").unwrap_or("/usr/local/bin/ansible-galaxy".to_string());
}
const NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT: &str = include_str!("../nsjail/run.ansible.config.proto");
async fn handle_ansible_python_deps(
job_dir: &str,
requirements_o: Option<String>,
@@ -98,6 +101,7 @@ async fn handle_ansible_python_deps(
}
Ok(additional_python_paths)
}
async fn install_galaxy_collections(
collections_yml: &str,
job_dir: &str,
@@ -155,6 +159,24 @@ async fn install_galaxy_collections(
Ok(())
}
#[cfg(not(feature = "enterprise"))]
fn check_ansible_exists() -> Result<(), error::Error> {
if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() {
let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str());
return Err(error::Error::NotFound(msg));
}
Ok(())
}
#[cfg(feature = "enterprise")]
fn check_ansible_exists() -> Result<(), error::Error> {
if !Path::new(ANSIBLE_PLAYBOOK_PATH.as_str()).exists() {
let msg = format!("Couldn't find ansible-playbook at {}. This probably means that you are not using the windmill-full image. Please use the image `windmill-full-ee` for your instance in order to run rust jobs.", ANSIBLE_PLAYBOOK_PATH.as_str());
return Err(error::Error::NotFound(msg));
}
Ok(())
}
pub async fn handle_ansible_job(
requirements_o: Option<String>,
job_dir: &str,
@@ -166,19 +188,16 @@ pub async fn handle_ansible_job(
db: &sqlx::Pool<sqlx::Postgres>,
client: &AuthedClientBackgroundTask,
inner_content: &String,
shared_mount: &str,
base_internal_url: &str,
envs: HashMap<String, String>,
) -> windmill_common::error::Result<Box<RawValue>> {
check_ansible_exists()?;
let (logs, reqs, playbook) = windmill_parser_yaml::parse_ansible_reqs(inner_content)?;
append_logs(&job.id, &job.workspace_id, logs, db).await;
write_file(job_dir, "main.yml", &playbook)?;
let ansible_cfg_content = r#"
[defaults]
collections_path = ./
roles_path = ./roles
"#;
write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?;
let additional_python_paths = handle_ansible_python_deps(
job_dir,
@@ -215,6 +234,7 @@ roles_path = ./roles
interpolated_args = None;
write_file(job_dir, "args.json", "{}")?;
};
write_file(job_dir, "result.json", "")?;
let inventories: Vec<String> = reqs
.as_ref()
@@ -228,8 +248,9 @@ roles_path = ./roles
.unwrap_or_else(|| vec![]);
let authed_client = client.get_authed().await;
let mut nsjail_extra_mounts = vec![];
if let Some(r) = reqs {
create_file_resources(
nsjail_extra_mounts = create_file_resources(
&job.id,
&job.workspace_id,
job_dir,
@@ -261,18 +282,97 @@ roles_path = ./roles
db,
)
.await;
if !*DISABLE_NSJAIL {
return Err(anyhow!("Ansible is not supported with nsjail, disable nsjail on your worker to run ansible playbooks").into());
}
let ansible_cfg_content = format!(r#"
[defaults]
collections_path = ./
roles_path = ./roles
home={job_dir}/.ansible
local_tmp={job_dir}/.ansible/tmp
remote_tmp={job_dir}/.ansible/tmp
"#);
write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?;
let mut reserved_variables = get_reserved_variables(job, &authed_client.token, db).await?;
let additional_python_paths_folders = additional_python_paths.join(":");
reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
if !*DISABLE_NSJAIL {
let shared_deps = additional_python_paths
.into_iter()
.map(|pp| {
format!(
r#"
mount {{
src: "{pp}"
dst: "{pp}"
is_bind: true
rw: false
}}
"#
)
})
.join("\n");
let _ = write_file(
job_dir,
"run.config.proto",
&NSJAIL_CONFIG_RUN_ANSIBLE_CONTENT
.replace("{JOB_DIR}", job_dir)
.replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string())
.replace("{SHARED_MOUNT}", shared_mount)
.replace("{SHARED_DEPENDENCIES}", shared_deps.as_str())
.replace("{FILE_RESOURCES}", nsjail_extra_mounts.join("\n").as_str())
.replace(
"{ADDITIONAL_PYTHON_PATHS}",
additional_python_paths_folders.as_str(),
),
)?;
} else {
reserved_variables.insert("PYTHONPATH".to_string(), additional_python_paths_folders);
}
let mut cmd_args = vec!["main.yml", "--extra-vars", "@args.json"];
cmd_args.extend(inventories.iter().map(|s| s.as_str()));
let child = {
let child = if !*DISABLE_NSJAIL {
let wrapper = format!(r#"set -eou pipefail
{0} "$@"
if [ -f "result" ]; then
cat result > result_nsjail_mount.json
fi
if [ -f "result.json" ]; then
cat result.json > result_nsjail_mount.json
fi
"#, ANSIBLE_PLAYBOOK_PATH.as_str());
let file = write_file(job_dir, "wrapper.sh", &wrapper)?;
file.metadata()?.permissions().set_mode(0o777);
// let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
let mut nsjail_cmd = Command::new(NSJAIL_PATH.as_str());
nsjail_cmd
.current_dir(job_dir)
.env_clear()
// inject PYTHONPATH here - for some reason I had to do it in nsjail conf
.envs(reserved_variables)
.env("PATH", PATH_ENV.as_str())
.env("TZ", TZ_ENV.as_str())
.env("BASE_INTERNAL_URL", base_internal_url)
.env("BASE_URL", base_internal_url)
.args(
vec![
"--config",
"run.config.proto",
"--",
BIN_BASH.as_str(),
"/tmp/wrapper.sh",
]
.into_iter()
.chain(cmd_args),
)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(nsjail_cmd, NSJAIL_PATH.as_str()).await?
} else {
let mut ansible_cmd = Command::new(ANSIBLE_PLAYBOOK_PATH.as_str());
ansible_cmd
.current_dir(job_dir)
@@ -306,6 +406,24 @@ roles_path = ./roles
read_and_check_result(job_dir).await
}
fn define_nsjail_mount(job_dir: &str, path: &PathBuf) -> anyhow::Result<String> {
Ok(format!(
r#"
mount {{
src: "{0}/{1}"
dst: "/tmp/{1}"
is_bind: true
rw: false
mandatory: false
}}
"#,
job_dir,
path.strip_prefix(job_dir)?
.to_str()
.ok_or(anyhow!("Invalid path."))?
))
}
async fn create_file_resources(
job_id: &Uuid,
w_id: &str,
@@ -314,8 +432,9 @@ async fn create_file_resources(
r: &AnsibleRequirements,
client: &crate::AuthedClient,
db: &sqlx::Pool<sqlx::Postgres>,
) -> error::Result<()> {
) -> error::Result<Vec<String>> {
let mut logs = String::new();
let mut nsjail_mounts: Vec<String> = vec![];
for inventory in &r.inventories {
let content;
@@ -336,9 +455,13 @@ async fn create_file_resources(
content = serde_json::from_str(o.get())
.map_err(|e| anyhow!("Failed to parse inventory arg: {}", e))?;
if content == serde_json::value::Value::Null {
Err(anyhow!("The inventory argument was left empty. If you do not wish to specify an inventory for this script, remove the `inventory:` section from the yaml."))?;
}
}
write_file_at_user_defined_location(
let validated_path = write_file_at_user_defined_location(
job_dir,
&inventory.name,
content
@@ -349,6 +472,12 @@ async fn create_file_resources(
))?,
)
.map_err(|e| anyhow!("Couldn't write inventory: {}", e))?;
nsjail_mounts.push(
define_nsjail_mount(job_dir, &validated_path)
.map_err(|e| anyhow!("Inventory path (a.k.a. `name`) is invalid: {}", e))?,
);
logs.push_str(&format!("\nCreated inventory `{}`", inventory.name));
}
@@ -360,7 +489,7 @@ async fn create_file_resources(
)
.await?;
let path = file_res.target_path.clone();
write_file_at_user_defined_location(
let validated_path = write_file_at_user_defined_location(
job_dir,
path.as_str(),
r.get("content").and_then(|v| v.as_str()).ok_or(anyhow!(
@@ -369,6 +498,12 @@ async fn create_file_resources(
))?,
)
.map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?;
nsjail_mounts.push(
define_nsjail_mount(job_dir, &validated_path)
.map_err(|e| anyhow!("File resource path is invalid: {}", e))?,
);
logs.push_str(&format!(
"\nCreated {} from {}",
file_res.target_path, file_res.resource_path
@@ -376,5 +511,5 @@ async fn create_file_resources(
}
append_logs(job_id, w_id, logs, db).await;
Ok(())
Ok(nsjail_mounts)
}
backend/windmill-worker/src/bash_executor.rs
+1 -1
View File
@@ -12,7 +12,7 @@ use windmill_common::{
use windmill_queue::{append_logs, CanceledBy};
lazy_static::lazy_static! {
static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string());
pub static ref BIN_BASH: String = std::env::var("BASH_PATH").unwrap_or_else(|_| "/bin/bash".to_string());
}
const NSJAIL_CONFIG_RUN_BASH_CONTENT: &str = include_str!("../nsjail/run.bash.config.proto");
const NSJAIL_CONFIG_RUN_POWERSHELL_CONTENT: &str =
backend/windmill-worker/src/bun_executor.rs
+95 -27
View File
@@ -14,7 +14,8 @@ use crate::common::build_envs_map;
use crate::{
common::{
create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
parse_npm_config, read_file_content, read_result, start_child_process, write_file_binary,
parse_npm_config, read_file, read_file_content, read_result, start_child_process,
write_file_binary,
},
AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH,
@@ -34,7 +35,7 @@ use windmill_common::variables;
use windmill_common::{
error::{self, Result},
get_latest_hash_for_path,
jobs::QueuedJob,
jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT},
scripts::ScriptLang,
worker::{exists_in_cache, get_annotation, save_cache, write_file},
DB,
@@ -69,8 +70,7 @@ pub async fn gen_bun_lockfile(
raw_deps: Option<String>,
npm_mode: bool,
) -> Result<Option<String>> {
let common_bun_proc_envs: HashMap<String, String> =
get_common_bun_proc_envs(&base_internal_url).await;
let common_bun_proc_envs: HashMap<String, String> = get_common_bun_proc_envs(None).await;
let mut empty_deps = false;
@@ -644,8 +644,7 @@ pub async fn prebundle_bun_script(
)
.await?;
let common_bun_proc_envs: HashMap<String, String> =
get_common_bun_proc_envs(&base_internal_url).await;
let common_bun_proc_envs: HashMap<String, String> = get_common_bun_proc_envs(None).await;
generate_bun_bundle(
job_dir,
@@ -751,6 +750,7 @@ pub async fn handle_bun_job(
worker_name: &str,
envs: HashMap<String, String>,
shared_mount: &str,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
) -> error::Result<Box<RawValue>> {
let mut annotation = windmill_common::worker::get_annotation(inner_content);
@@ -779,12 +779,21 @@ pub async fn handle_bun_job(
};
let common_bun_proc_envs: HashMap<String, String> =
get_common_bun_proc_envs(&base_internal_url).await;
get_common_bun_proc_envs(Some(&base_internal_url)).await;
if codebase.is_some() {
annotation.nodejs_mode = true
}
let main_override = get_main_override(job.args.as_ref());
let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) {
Some(main_override) => {
if main_override == PREPROCESSOR_FAKE_ENTRYPOINT {
(None, true)
} else {
(Some(main_override), false)
}
}
None => (None, false),
};
#[cfg(not(feature = "enterprise"))]
if annotation.nodejs_mode || annotation.npm_mode {
@@ -939,18 +948,35 @@ pub async fn handle_bun_job(
let args =
windmill_parser_ts::parse_deno_signature(inner_content, true, main_override.clone())?
.args;
let dates = args
let pre_args = if apply_preprocessor {
Some(
windmill_parser_ts::parse_deno_signature(
inner_content,
true,
Some("preprocessor".to_string()),
)?
.args,
)
} else {
None
};
let dates = pre_args
.as_ref()
.unwrap_or(&args)
.iter()
.enumerate()
.filter_map(|(i, x)| {
.filter_map(|x| {
if matches!(x.typ, Typ::Datetime) {
Some(i)
Some(x.name.as_str())
} else {
None
}
})
.map(|x| return format!("args[{x}] = args[{x}] ? new Date(args[{x}]) : undefined"))
.join("\n");
.map(|x| {
return format!(r#"args["{x}"] = args["{x}"] ? new Date(args["{x}"]) : undefined"#);
})
.join("\n ");
let spread = args.into_iter().map(|x| x.name).join(",");
// logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str());
@@ -963,22 +989,47 @@ pub async fn handle_bun_job(
"./main.ts"
};
let wrapper_content: String = format!(
let preprocessor = if let Some(pre_args) = pre_args {
let pre_spread = pre_args.into_iter().map(|x| x.name).join(",");
format!(
r#"if (Main.preprocessor === undefined || typeof Main.preprocessor !== 'function') {{
throw new Error("preprocessor function is missing");
}}
function preArgsObjToArr({{ {pre_spread} }}) {{
return [ {pre_spread} ];
}}
args = await Main.preprocessor(...preArgsObjToArr(args));
const args_json = JSON.stringify(args ?? null, (key, value) => typeof value === 'undefined' ? null : value);
await fs.writeFile('args.json', args_json, {{ encoding: 'utf8' }})"#
)
} else {
"".to_string()
};
let wrapper_content = format!(
r#"
import * as Main from "{main_import}";
import * as fs from "fs/promises";
const args = await fs.readFile('args.json', {{ encoding: 'utf8' }}).then(JSON.parse)
.then(({{ {spread} }}) => [ {spread} ])
let args = await fs.readFile('args.json', {{ encoding: 'utf8' }}).then(JSON.parse);
function argsObjToArr({{ {spread} }}) {{
return [ {spread} ];
}}
BigInt.prototype.toJSON = function () {{
return this.toString();
}};
{dates}
async function run() {{
let res = await Main.{main_name}(...args);
{dates}
{preprocessor}
const argsArr = argsObjToArr(args);
if (Main.{main_name} === undefined || typeof Main.{main_name} !== 'function') {{
throw new Error("{main_name} function is missing");
}}
let res = await Main.{main_name}(...argsArr);
const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
await fs.writeFile("result.json", res_json);
process.exit(0);
@@ -1311,22 +1362,33 @@ try {{
false,
)
.await?;
if apply_preprocessor {
let args = read_file(&format!("{job_dir}/args.json"))
.await
.map_err(|e| {
error::Error::InternalErr(format!(
"error while reading args from preprocessing: {e:#}"
))
})?;
let args: HashMap<String, Box<RawValue>> =
serde_json::from_str(args.get()).map_err(|e| {
error::Error::InternalErr(format!(
"error while deserializing args from preprocessing: {e:#}"
))
})?;
*new_args = Some(args.clone());
}
read_result(job_dir).await
}
pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String, String> {
pub async fn get_common_bun_proc_envs(base_internal_url: Option<&str>) -> HashMap<String, String> {
let mut bun_envs: HashMap<String, String> = HashMap::from([
(String::from("PATH"), PATH_ENV.clone()),
(String::from("HOME"), HOME_ENV.clone()),
(String::from("TZ"), TZ_ENV.clone()),
(String::from("FORCE_COLOR"), "1".to_string()),
(String::from("DO_NOT_TRACK"), "1".to_string()),
(
String::from("BASE_URL"),
base_internal_url
.to_string()
.replace("localhost", "127.0.0.1"),
),
(
String::from("BUN_INSTALL_CACHE_DIR"),
BUN_CACHE_DIR.to_string(),
@@ -1337,6 +1399,12 @@ pub async fn get_common_bun_proc_envs(base_internal_url: &str) -> HashMap<String
),
]);
if let Some(base_url) = base_internal_url {
bun_envs.insert(
String::from("BASE_URL"),
base_url.to_string().replace("localhost", "127.0.0.1"),
);
}
if let Some(ref node_path) = NODE_PATH.as_ref() {
bun_envs.insert(String::from("NODE_PATH"), node_path.to_string());
}
@@ -1373,7 +1441,7 @@ pub async fn start_worker(
}
let common_bun_proc_envs: HashMap<String, String> =
get_common_bun_proc_envs(&base_internal_url).await;
get_common_bun_proc_envs(Some(&base_internal_url)).await;
let mut annotation = windmill_common::worker::get_annotation(inner_content);
backend/windmill-worker/src/common.rs
+3 -3
View File
@@ -107,7 +107,7 @@ pub async fn create_args_and_out_file(
job_dir: &str,
db: &Pool<Postgres>,
) -> Result<(), Error> {
if let Some(args) = &job.args {
if let Some(args) = job.args.as_ref() {
if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, db).await? {
write_file(
job_dir,
@@ -380,8 +380,8 @@ pub async fn read_and_check_file(path: &str) -> error::Result<Box<RawValue>> {
check_result_too_big(content.len())?;
let raw_value: Box<RawValue> = serde_json::from_str(&content)
.map_err(|e| anyhow!("{} is not valid json: {}", path, e))?;
let raw_value: Box<RawValue> =
serde_json::from_str(&content).map_err(|e| anyhow!("{} is not valid json: {}", path, e))?;
Ok(raw_value)
}
backend/windmill-worker/src/deno_executor.rs
+88 -14
View File
@@ -8,13 +8,15 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
parse_npm_config, read_result, start_child_process,
parse_npm_config, read_file, read_result, start_child_process,
},
AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV,
NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV,
};
use tokio::{fs::File, io::AsyncReadExt, process::Command};
use windmill_common::{error::Result, worker::write_file, BASE_URL};
use windmill_common::{
error::Result, jobs::PREPROCESSOR_FAKE_ENTRYPOINT, worker::write_file, BASE_URL,
};
use windmill_common::{
error::{self},
jobs::QueuedJob,
@@ -170,12 +172,22 @@ pub async fn handle_deno_job(
base_internal_url: &str,
worker_name: &str,
envs: HashMap<String, String>,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
) -> error::Result<Box<RawValue>> {
// let mut start = Instant::now();
let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string();
append_logs(&job.id, &job.workspace_id, logs1, db).await;
let main_override = get_main_override(job.args.as_ref());
let (main_override, apply_preprocessor) = match get_main_override(job.args.as_ref()) {
Some(main_override) => {
if main_override == PREPROCESSOR_FAKE_ENTRYPOINT {
(None, true)
} else {
(Some(main_override), false)
}
}
None => (None, false),
};
write_file(job_dir, "main.ts", inner_content)?;
@@ -184,37 +196,83 @@ pub async fn handle_deno_job(
let args =
windmill_parser_ts::parse_deno_signature(inner_content, true, main_override.clone())?
.args;
let dates = args
let pre_args = if apply_preprocessor {
Some(
windmill_parser_ts::parse_deno_signature(
inner_content,
true,
Some("preprocessor".to_string()),
)?
.args,
)
} else {
None
};
let dates = pre_args
.as_ref()
.unwrap_or(&args)
.iter()
.enumerate()
.filter_map(|(i, x)| {
.filter_map(|x| {
if matches!(x.typ, Typ::Datetime) {
Some(i)
Some(x.name.as_str())
} else {
None
}
})
.map(|x| return format!("args[{x}] = args[{x}] ? new Date(args[{x}]) : undefined"))
.join("\n");
.map(|x| {
return format!(r#"args["{x}"] = args["{x}"] ? new Date(args["{x}"]) : undefined"#);
})
.join("\n ");
let spread = args.into_iter().map(|x| x.name).join(",");
let main_name = main_override.unwrap_or("main".to_string());
// logs.push_str(format!("infer args: {:?}\n", start.elapsed().as_micros()).as_str());
let (preprocessor_import, preprocessor) = if let Some(pre_args) = pre_args {
let pre_spread = pre_args.into_iter().map(|x| x.name).join(",");
(
r#"import { preprocessor } from "./main.ts";"#.to_string(),
format!(
r#"if (preprocessor === undefined || typeof preprocessor !== 'function') {{
throw new Error("preprocessor function is missing");
}}
function preArgsObjToArr({{ {pre_spread} }}) {{
return [ {pre_spread} ];
}}
args = await preprocessor(...preArgsObjToArr(args));
const args_json = JSON.stringify(args ?? null, (key, value) => typeof value === 'undefined' ? null : value);
await Deno.writeTextFile("args.json", args_json);"#
),
)
} else {
("".to_string(), "".to_string())
};
let wrapper_content: String = format!(
r#"
import {{ {main_name} }} from "./main.ts";
{preprocessor_import}
const args = await Deno.readTextFile("args.json")
.then(JSON.parse)
.then(({{ {spread} }}) => [ {spread} ])
let args = await Deno.readTextFile("args.json")
.then(JSON.parse);
function argsObjToArr({{ {spread} }}) {{
return [ {spread} ];
}}
BigInt.prototype.toJSON = function () {{
return this.toString();
}};
{dates}
async function run() {{
let res: any = await {main_name}(...args);
{dates}
{preprocessor}
const argsArr = argsObjToArr(args);
if ({main_name} === undefined || typeof {main_name} !== 'function') {{
throw new Error("{main_name} function is missing");
}}
let res: any = await {main_name}(...argsArr);
const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value);
await Deno.writeTextFile("result.json", res_json);
Deno.exit(0);
@@ -346,6 +404,22 @@ try {{
{
tracing::error!("failed to remove deno gen tmp cache dir: {}", e);
}
if apply_preprocessor {
let args = read_file(&format!("{job_dir}/args.json"))
.await
.map_err(|e| {
error::Error::InternalErr(format!(
"error while reading args from preprocessing: {e:#}"
))
})?;
let args: HashMap<String, Box<RawValue>> =
serde_json::from_str(args.get()).map_err(|e| {
error::Error::InternalErr(format!(
"error while deserializing args from preprocessing: {e:#}"
))
})?;
*new_args = Some(args.clone());
}
read_result(job_dir).await
}
backend/windmill-worker/src/python_executor.rs
+125 -24
View File
@@ -14,7 +14,7 @@ use uuid::Uuid;
use windmill_common::ee::{get_license_plan, LicensePlan};
use windmill_common::{
error::{self, Error},
jobs::QueuedJob,
jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT},
utils::calculate_hash,
worker::{write_file, WORKER_CONFIG},
DB,
@@ -56,7 +56,7 @@ use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
use crate::{
common::{
create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
read_result, start_child_process,
read_file, read_result, start_child_process,
},
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY,
LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL,
@@ -246,6 +246,7 @@ pub async fn handle_python_job(
shared_mount: &str,
base_internal_url: &str,
envs: HashMap<String, String>,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
) -> windmill_common::error::Result<Box<RawValue>> {
let script_path = crate::common::use_flow_root_path(job.script_path());
let additional_python_paths = handle_python_deps(
@@ -281,10 +282,39 @@ pub async fn handle_python_job(
transforms,
spread,
main_name,
) = prepare_wrapper(job_dir, inner_content, &script_path, job.args.as_ref()).await?;
pre_spread,
) = prepare_wrapper(
job_dir,
inner_content,
&script_path,
job.args.as_ref(),
false,
)
.await?;
let apply_preprocessor = pre_spread.is_some();
create_args_and_out_file(&client, job, job_dir, db).await?;
let preprocessor = if let Some(pre_spread) = pre_spread {
format!(
r#"if inner_script.preprocessor is None or not callable(inner_script.preprocessor):
raise ValueError("preprocessor function is missing")
else:
pre_args = {{}}
{pre_spread}
for k, v in list(pre_args.items()):
if v == '<function call>':
del pre_args[k]
kwargs = inner_script.preprocessor(**pre_args)
kwrags_json = res_to_json(kwargs)
with open("args.json", 'w') as f:
f.write(kwrags_json)"#
)
} else {
"".to_string()
};
let os_main_override = if let Some(main_override) = main_name.as_ref() {
format!("os.environ[\"MAIN_OVERRIDE\"] = \"{main_override}\"\n")
} else {
@@ -308,10 +338,6 @@ with open("args.json") as f:
kwargs = json.load(f, strict=False)
args = {{}}
{transforms}
{spread}
for k, v in list(args.items()):
if v == '<function call>':
del args[k]
def to_b_64(v: bytes):
import base64
@@ -322,8 +348,7 @@ replace_nan = re.compile(r'(?:\bNaN\b|\\*\\u0000)')
result_json = os.path.join(os.path.abspath(os.path.dirname(__file__)), "result.json")
try:
res = inner_script.{main_override}(**args)
def res_to_json(res):
typ = type(res)
if typ.__name__ == 'DataFrame':
if typ.__module__ == 'pandas.core.frame':
@@ -336,7 +361,18 @@ try:
for k, v in res.items():
if type(v).__name__ == 'bytes':
res[k] = to_b_64(v)
res_json = re.sub(replace_nan, ' null ', json.dumps(res, separators=(',', ':'), default=str).replace('\n', ''))
return re.sub(replace_nan, ' null ', json.dumps(res, separators=(',', ':'), default=str).replace('\n', ''))
try:
{preprocessor}
{spread}
for k, v in list(args.items()):
if v == '<function call>':
del args[k]
if inner_script.{main_override} is None or not callable(inner_script.{main_override}):
raise ValueError("{main_override} function is missing")
res = inner_script.{main_override}(**args)
res_json = res_to_json(res)
with open(result_json, 'w') as f:
f.write(res_json)
except BaseException as e:
@@ -451,6 +487,24 @@ mount {{
false,
)
.await?;
if apply_preprocessor {
let args = read_file(&format!("{job_dir}/args.json"))
.await
.map_err(|e| {
error::Error::InternalErr(format!(
"error while reading args from preprocessing: {e:#}"
))
})?;
let args: HashMap<String, Box<RawValue>> =
serde_json::from_str(args.get()).map_err(|e| {
error::Error::InternalErr(format!(
"error while deserializing args from preprocessing: {e:#}"
))
})?;
*new_args = Some(args.clone());
}
read_result(job_dir).await
}
@@ -459,6 +513,7 @@ async fn prepare_wrapper(
inner_content: &str,
script_path: &str,
args: Option<&Json<HashMap<String, Box<RawValue>>>>,
skip_preprocessor: bool,
) -> error::Result<(
&'static str,
&'static str,
@@ -469,8 +524,18 @@ async fn prepare_wrapper(
String,
String,
Option<String>,
Option<String>,
)> {
let main_override = get_main_override(args);
let (main_override, apply_preprocessor) = match get_main_override(args) {
Some(main_override) => {
if !skip_preprocessor && main_override == PREPROCESSOR_FAKE_ENTRYPOINT {
(None, true)
} else {
(Some(main_override), false)
}
}
None => (None, false),
};
let relative_imports = RELATIVE_IMPORT_REGEX.is_match(&inner_content);
@@ -511,7 +576,20 @@ async fn prepare_wrapper(
}
let sig = windmill_parser_py::parse_python_signature(inner_content, main_override.clone())?;
let transforms = sig
let pre_sig = if apply_preprocessor {
Some(windmill_parser_py::parse_python_signature(
inner_content,
Some("preprocessor".to_string()),
)?)
} else {
None
};
// transforms should be applied based on the signature of the first function called
let init_sig = pre_sig.as_ref().unwrap_or(&sig);
let transforms = init_sig
.args
.iter()
.map(|x| match x.typ {
@@ -539,7 +617,7 @@ async fn prepare_wrapper(
} else {
""
};
let import_base64 = if sig
let import_base64 = if init_sig
.args
.iter()
.any(|x| x.typ == windmill_parser::Typ::Bytes)
@@ -548,7 +626,7 @@ async fn prepare_wrapper(
} else {
""
};
let import_datetime = if sig
let import_datetime = if init_sig
.args
.iter()
.any(|x| x.typ == windmill_parser::Typ::Datetime)
@@ -569,12 +647,38 @@ async fn prepare_wrapper(
} else {
format!(
r#"args["{name}"] = kwargs.get("{name}")
if args["{name}"] is None:
del args["{name}"]"#
if args["{name}"] is None:
del args["{name}"]"#
)
}
})
.join("\n")
.join("\n ")
};
let pre_spread = if let Some(pre_sig) = pre_sig {
let spread = if pre_sig.star_kwargs {
"pre_args = kwargs".to_string()
} else {
pre_sig
.args
.into_iter()
.map(|x| {
let name = &x.name;
if x.default.is_none() {
format!("pre_args[\"{name}\"] = kwargs.get(\"{name}\")")
} else {
format!(
r#"pre_args["{name}"] = kwargs.get("{name}")
if pre_args["{name}"] is None:
del pre_args["{name}"]"#
)
}
})
.join("\n ")
};
Some(spread)
} else {
None
};
let module_dir_dot = dirs.replace("/", ".").replace("-", "_");
@@ -589,6 +693,7 @@ if args["{name}"] is None:
transforms,
spread,
main_override,
pre_spread,
))
}
@@ -1082,7 +1187,8 @@ pub async fn start_worker(
transforms,
spread,
_,
) = prepare_wrapper(job_dir, inner_content, script_path, _args.as_ref()).await?;
_,
) = prepare_wrapper(job_dir, inner_content, script_path, _args.as_ref(), true).await?;
{
let indented_transforms = transforms
@@ -1090,11 +1196,6 @@ pub async fn start_worker(
.map(|x| format!(" {}", x))
.collect::<Vec<String>>()
.join("\n");
let indented_spread = spread
.lines()
.map(|x| format!(" {}", x))
.collect::<Vec<String>>()
.join("\n");
let wrapper_content: String = format!(
r#"
@@ -1122,7 +1223,7 @@ for line in sys.stdin:
kwargs = json.loads(line, strict=False)
args = {{}}
{indented_transforms}
{indented_spread}
{spread}
for k, v in list(args.items()):
if v == '<function call>':
del args[k]
backend/windmill-worker/src/result_processor.rs
+38 -15
View File
@@ -1,6 +1,6 @@
use serde::Serialize;
use sqlx::{types::Json, Pool, Postgres};
use std::sync::Arc;
use std::{collections::HashMap, sync::Arc};
use uuid::Uuid;
@@ -53,26 +53,49 @@ pub async fn process_result(
cached_res_path: Option<String>,
token: String,
column_order: Option<Vec<String>>,
new_args: Option<HashMap<String, Box<RawValue>>>,
db: &DB,
) -> error::Result<()> {
match result {
Ok(r) => {
let job = if let Some(column_order) = column_order {
let mut job_with_column_order = (*job).clone();
match job_with_column_order.flow_status {
Some(_) => {
tracing::warn!("flow_status was expected to be none");
}
None => {
job_with_column_order.flow_status =
Some(sqlx::types::Json(to_raw_value(&serde_json::json!({
"_metadata": {
"column_order": column_order
}
}))));
let job = if column_order.is_some() || new_args.is_some() {
let mut updated_job = (*job).clone();
if let Some(column_order) = column_order {
match updated_job.flow_status {
Some(_) => {
tracing::warn!("flow_status was expected to be none");
}
None => {
updated_job.flow_status =
Some(sqlx::types::Json(to_raw_value(&serde_json::json!({
"_metadata": {
"column_order": column_order
}
}))));
}
}
}
Arc::new(job_with_column_order)
if let Some(new_args) = new_args {
match updated_job.flow_status {
Some(_) => {
tracing::warn!("flow_status was expected to be none");
}
None => {
// TODO save original args somewhere
// if let Some(args) = updated_job.args.as_mut() {
// args.0.remove(ENTRYPOINT_OVERRIDE);
// }
updated_job.flow_status =
Some(sqlx::types::Json(to_raw_value(&serde_json::json!({
"_metadata": {
"preprocessed_args": true
}
}))));
}
}
updated_job.args = Some(Json(new_args));
}
Arc::new(updated_job)
} else {
job
};
backend/windmill-worker/src/worker.rs
+19 -7
View File
@@ -85,9 +85,9 @@ use rand::Rng;
use crate::{
ansible_executor::handle_ansible_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{
build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args,
NO_LOGS_AT_ALL, SLOW_LOGS,
}, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{handle_job_error, handle_receive_completed_job, process_result}, rust_executor::handle_rust_job, worker_flow::{
handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress,
NO_LOGS_AT_ALL, SLOW_LOGS,
}, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, handle_job_error, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{handle_receive_completed_job, process_result}, rust_executor::handle_rust_job, worker_flow::{
handle_flow, update_flow_status_after_job_completion, update_flow_status_in_progress, Step,
}, worker_lockfiles::{
handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job,
}
@@ -2050,7 +2050,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
}
#[cfg(any(not(feature = "enterprise"), feature = "sqlx"))]
if job.created_by.starts_with("email-trigger-") {
if job.created_by.starts_with("email-") {
let daily_count = sqlx::query!(
"SELECT value FROM metrics WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day' ORDER BY created_at DESC LIMIT 1"
).fetch_optional(db).await?.map(|x| serde_json::from_value::<i64>(x.value).unwrap_or(1));
@@ -2087,7 +2087,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
)
.await?;
r
Some(r)
} else {
if let Some(parent_job) = job.parent_job {
if let Err(e) = sqlx::query_scalar!(
@@ -2139,7 +2139,11 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
.await
.map_err(|e| Error::InternalErr(format!("fetching step flow status: {e:#}")))?
.ok_or_else(|| Error::InternalErr(format!("Expected script_path")))?;
let step = step.unwrap_or(-1);
let step = match step.unwrap() {
Step::Step(i) => i.to_string(),
Step::PreprocessorStep => "preprocessor".to_string(),
Step::FailureStep => "failure".to_string(),
};
Some(format!(
"{flow_path}/cache/{version_hash}/{step}/{args_hash}"
))
@@ -2239,6 +2243,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
append_logs(&job.id, &job.workspace_id, logs, db).await;
let mut column_order: Option<Vec<String>> = None;
let mut new_args: Option<HashMap<String, Box<RawValue>>> = None;
let result = match job.job_kind {
JobKind::Dependencies => {
handle_dependency_job(
@@ -2306,6 +2311,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
base_internal_url,
worker_name,
&mut column_order,
&mut new_args,
)
.await;
*worker_code_execution_metric += metric_timer.elapsed().as_secs_f32();
@@ -2337,6 +2343,7 @@ async fn handle_queued_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>(
cached_res_path,
client.get_token().await,
column_order,
new_args,
db,
)
.await?;
@@ -2432,7 +2439,7 @@ pub async fn get_script_content_by_hash(
Option<String>,
Option<ScriptLang>,
Option<Vec<String>>,
Option<bool>,
Option<bool>
),
>(
"SELECT content, lock, language, envs, codebase LIKE '%.tar' as codebase FROM script WHERE hash = $1 AND workspace_id = $2",
@@ -2473,6 +2480,7 @@ async fn handle_code_execution_job(
base_internal_url: &str,
worker_name: &str,
column_order: &mut Option<Vec<String>>,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
) -> error::Result<Box<RawValue>> {
let ContentReqLangEnvs {
content: inner_content,
@@ -2705,6 +2713,7 @@ mount {{
&shared_mount,
base_internal_url,
envs,
new_args,
)
.await
}
@@ -2721,6 +2730,7 @@ mount {{
base_internal_url,
worker_name,
envs,
new_args,
)
.await
}
@@ -2739,6 +2749,7 @@ mount {{
worker_name,
envs,
&shared_mount,
new_args,
)
.await
}
@@ -2838,6 +2849,7 @@ mount {{
db,
client,
&inner_content,
&shared_mount,
base_internal_url,
envs,
).await
backend/windmill-worker/src/worker_flow.rs
+242 -113
View File
@@ -37,7 +37,7 @@ use windmill_common::flow_status::{
use windmill_common::flows::add_virtual_items_if_necessary;
use windmill_common::jobs::{
script_hash_to_tag_and_limits, script_path_to_payload, BranchResults, JobPayload, QueuedJob,
RawCode,
RawCode, ENTRYPOINT_OVERRIDE,
};
use windmill_common::worker::to_raw_value;
use windmill_common::{
@@ -225,14 +225,22 @@ pub async fn update_flow_status_after_job_completion_internal<
)))
})?;
let module_index = usize::try_from(old_status.step).ok();
let module_step = Step::from_i32_and_len(old_status.step, old_status.modules.len());
let module_status = module_index
.and_then(|i| old_status.modules.get(i))
.unwrap_or(&old_status.failure_module.module_status);
let module_status = match module_step {
Step::PreprocessorStep => old_status
.preprocessor_module
.as_ref()
.ok_or_else(|| Error::InternalErr(format!("preprocessor module not found")))?,
Step::FailureStep => &old_status.failure_module.module_status,
Step::Step(i) => old_status
.modules
.get(i as usize)
.ok_or_else(|| Error::InternalErr(format!("module {i} not found")))?,
};
// tracing::debug!(
// "UPDATE FLOW STATUS 2: {module_index:#?} {module_status:#?} {old_status:#?} "
// "UPDATE FLOW STATUS 2: {module_step:#?} {module_status:#?} {old_status:#?} "
// );
let (is_loop, skip_loop_failures, parallelism) = if matches!(
@@ -262,14 +270,14 @@ pub async fn update_flow_status_after_job_completion_internal<
let flow_job = get_queued_job(&flow, w_id, db)
.await?
.ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
let module = get_module(&flow_job, module_index);
let module = get_module(&flow_job, &module_step);
if module.is_some_and(|x| x.is_flow()) {
(false, false, false)
} else {
(true, se, false)
}
} else if is_failure_step {
} else if is_failure_step || matches!(module_step, Step::PreprocessorStep) {
(false, false, false)
} else {
let r = sqlx::query_as::<_, SkipIfStopped>(
@@ -346,6 +354,28 @@ pub async fn update_flow_status_after_job_completion_internal<
let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
if matches!(module_step, Step::PreprocessorStep) {
sqlx::query!(
"UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2",
job_id_for_status,
flow
).execute(&mut tx).await.map_err(|e| {
Error::InternalErr(format!("error while updating args in preprocessing step: {e:#}"))
})?;
sqlx::query!(
r#"UPDATE completed_job SET args = '{"reason":"PREPROCESSOR_ARGS_ARE_DISCARDED"}'::jsonb WHERE id = $1"#,
job_id_for_status
)
.execute(&mut tx)
.await
.map_err(|e| {
Error::InternalErr(format!(
"error while deleting args of preprocessing step: {e:#}"
))
})?;
}
let (inc_step_counter, new_status) = match module_status {
FlowStatusModule::InProgress {
iterator,
@@ -713,6 +743,21 @@ pub async fn update_flow_status_after_job_completion_internal<
"error while setting flow status in failure step: {e:#}"
))
})?;
} else if matches!(module_step, Step::PreprocessorStep) {
sqlx::query!(
"UPDATE queue
SET flow_status = JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1)
WHERE id = $2",
json!(new_status),
flow
)
.execute(&mut tx)
.await
.map_err(|e| {
Error::InternalErr(format!(
"error while setting flow status in preprocessing step: {e:#}"
))
})?;
} else {
sqlx::query!(
"UPDATE queue
@@ -824,8 +869,7 @@ pub async fn update_flow_status_after_job_completion_internal<
.unwrap_or_else(|| "none".to_string());
tracing::info!(id = %flow_job.id, root_id = %job_root, "update flow status");
let module = get_module(&flow_job, module_index);
let module = get_module(&flow_job, &module_step);
// tracing::error!(
// "UPDATE FLOW STATUS 3: {module:#?} {unrecoverable} {} {is_last_step} {success} {skip_error_handler} is_failure_step {is_failure_step}", flow_job.canceled
// );
@@ -1115,17 +1159,13 @@ async fn retrieve_flow_jobs_results(
Ok(to_raw_value(&results))
}
fn get_module(flow_job: &QueuedJob, module_index: Option<usize>) -> Option<FlowModule> {
fn get_module(flow_job: &QueuedJob, module_step: &Step) -> Option<FlowModule> {
let raw_flow = flow_job.parse_raw_flow();
if let Some(raw_flow) = raw_flow {
if let Some(i) = module_index {
if let Some(module) = raw_flow.modules.get(i) {
Some(module.clone())
} else {
raw_flow.failure_module.map(|x| *x.clone())
}
} else {
None
match module_step {
Step::PreprocessorStep => raw_flow.preprocessor_module.map(|x| *x.clone()),
Step::Step(i) => raw_flow.modules.get(*i).map(|x| x.clone()),
Step::FailureStep => raw_flow.failure_module.map(|x| *x.clone()),
}
} else {
None
@@ -1293,56 +1333,86 @@ pub async fn update_flow_status_in_progress(
w_id: &str,
flow: Uuid,
job_in_progress: Uuid,
) -> error::Result<Option<i32>> {
) -> error::Result<Step> {
let step = get_step_of_flow_status(db, flow).await?;
if let Step::Step(step) = step {
sqlx::query(&format!(
"UPDATE queue
SET flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {step}, job}}', $1), '{{modules, {step}, type}}', $2)
WHERE id = $3 AND workspace_id = $4",
))
.bind(json!(job_in_progress.to_string()))
.bind(json!("InProgress"))
.bind(flow)
.bind(w_id)
.execute(db)
.await?;
Ok(Some(step))
} else {
sqlx::query(&format!(
"UPDATE queue
SET flow_status = jsonb_set(jsonb_set(flow_status, '{{failure_module, job}}', $1), '{{failure_module, type}}', $2)
WHERE id = $3 AND workspace_id = $4",
))
.bind(json!(job_in_progress.to_string()))
.bind(json!("InProgress"))
.bind(flow)
.bind(w_id)
.execute(db)
.await?;
Ok(None)
match step {
Step::Step(step) => {
sqlx::query(&format!(
"UPDATE queue
SET flow_status = jsonb_set(jsonb_set(flow_status, '{{modules, {step}, job}}', $1), '{{modules, {step}, type}}', $2)
WHERE id = $3 AND workspace_id = $4",
))
.bind(json!(job_in_progress.to_string()))
.bind(json!("InProgress"))
.bind(flow)
.bind(w_id)
.execute(db)
.await?;
}
Step::PreprocessorStep => {
sqlx::query(&format!(
"UPDATE queue
SET flow_status = jsonb_set(jsonb_set(flow_status, '{{preprocessor_module, job}}', $1), '{{preprocessor_module, type}}', $2)
WHERE id = $3 AND workspace_id = $4",
))
.bind(json!(job_in_progress.to_string()))
.bind(json!("InProgress"))
.bind(flow)
.bind(w_id)
.execute(db)
.await?;
}
Step::FailureStep => {
sqlx::query(&format!(
"UPDATE queue
SET flow_status = jsonb_set(jsonb_set(flow_status, '{{failure_module, job}}', $1), '{{failure_module, type}}', $2)
WHERE id = $3 AND workspace_id = $4",
))
.bind(json!(job_in_progress.to_string()))
.bind(json!("InProgress"))
.bind(flow)
.bind(w_id)
.execute(db)
.await?;
}
}
Ok(step)
}
#[derive(Debug)]
pub enum Step {
Step(usize),
PreprocessorStep,
FailureStep,
}
impl Step {
fn from_i32_and_len(step: i32, len: usize) -> Self {
if step < 0 {
Step::PreprocessorStep
} else if (step as usize) < len {
Step::Step(step as usize)
} else {
Step::FailureStep
}
}
}
pub enum Step {
Step(i32),
FailureStep,
}
#[instrument(level = "trace", skip_all)]
pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result<Step> {
let r = sqlx::query!(
"SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1",
"SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len FROM queue WHERE id = $1",
id
)
.fetch_one(db)
.await
.map_err(|e| Error::InternalErr(format!("fetching step flow status: {e:#}")))?;
if r.step < r.len {
Ok(Step::Step(r.step.ok_or_else(|| {
Error::InternalErr("step is null".to_string())
})?))
if let Some(step) = r.step {
Ok(Step::from_i32_and_len(step, r.len.unwrap_or(0) as usize))
} else {
Ok(Step::FailureStep)
Err(Error::InternalErr("step is null".to_string()))
}
}
@@ -1539,14 +1609,20 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
.unwrap_or_else(|| "none".to_string());
tracing::info!(id = %flow_job.id, root_id = %job_root, "pushing next flow job");
let mut i = usize::try_from(status.step)
.with_context(|| format!("invalid module index {}", status.step))?;
let mut step = Step::from_i32_and_len(status.step, flow.modules.len());
let mut status_module: FlowStatusModule = status
.modules
.get(i)
.cloned()
.unwrap_or_else(|| status.failure_module.module_status.clone());
let mut status_module = match step {
Step::Step(i) => status
.modules
.get(i)
.cloned()
.unwrap_or_else(|| status.failure_module.module_status.clone()),
Step::PreprocessorStep => status
.preprocessor_module
.clone()
.unwrap_or_else(|| status.failure_module.module_status.clone()),
Step::FailureStep => status.failure_module.module_status.clone(),
};
let fj: mappable_rc::Marc<QueuedJob> = flow_job.clone().into();
let arc_flow_job_args: Marc<HashMap<String, Box<RawValue>>> = Marc::map(fj, |x| {
@@ -1584,7 +1660,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
return Ok(());
}
if i == 0 {
if matches!(step, Step::Step(0)) {
if !flow_job.is_flow_step && flow_job.schedule_path.is_some() {
let no_flow_overlap = sqlx::query_scalar!(
"SELECT no_flow_overlap FROM schedule WHERE path = $1 AND workspace_id = $2",
@@ -1672,7 +1748,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
let arc_last_job_result = if status_module.is_failure() {
// if job is being retried, pass the result of its previous failure
last_job_result.unwrap_or_else(|| Arc::new(to_raw_value(&json!("{}"))))
} else if i == 0 {
} else if matches!(step, Step::Step(0)) || matches!(step, Step::PreprocessorStep) {
// if it's the first job executed in the flow, pass the flow args
Arc::new(to_raw_value(&flow_job.args))
} else {
@@ -1929,17 +2005,25 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
}
}
let mut module: &FlowModule = flow
.modules
.get(i)
.or_else(|| flow.failure_module.as_deref())
.with_context(|| format!("no module at index {}", status.step))?;
let mut module = match step {
Step::Step(i) => flow
.modules
.get(i)
.with_context(|| format!("no module at index {}", i))?,
Step::PreprocessorStep => flow
.preprocessor_module
.as_ref()
.with_context(|| format!("no preprocessor module"))?,
Step::FailureStep => flow
.failure_module
.as_deref()
.with_context(|| format!("no failure module"))?,
};
let current_id = &module.id;
let mut previous_id = if i >= 1 {
flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap()
} else {
String::new()
let mut previous_id = match step {
Step::Step(i) if i >= 1 => flow.modules.get(i - 1).map(|m| m.id.clone()).unwrap(),
_ => String::new(),
};
// calculate sleep if any
@@ -1952,10 +2036,13 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
) {
None
} else {
let sleep_input_transform = i
.checked_sub(1)
.and_then(|i| flow.modules.get(i))
.and_then(|m| m.sleep.clone());
let sleep_input_transform = if let Step::Step(i) = step {
i.checked_sub(1)
.and_then(|i| flow.modules.get(i))
.and_then(|m| m.sleep.clone())
} else {
None
};
if let Some(it) = sleep_input_transform {
let json_value = match it {
@@ -2054,7 +2141,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
* The failure module may also run again if it fails and the retry feature is used.
* In that case, `i` will index past `flow.modules`. The above should handle that and
* re-run the failure module. */
i = flow.modules.len();
step = Step::FailureStep;
previous_id = current_id.clone();
module = flow
.failure_module
@@ -2121,6 +2208,13 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
} else {
Ok(Marc::new(HashMap::new()))
}
} else if matches!(step, Step::PreprocessorStep) {
let mut hm = (*arc_flow_job_args).clone();
hm.insert(
ENTRYPOINT_OVERRIDE.to_string(),
to_raw_value(&"preprocessor"),
);
Ok(Marc::new(hm))
} else {
match &module.get_value() {
Ok(
@@ -2522,6 +2616,7 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
tracing::debug!(id = %flow_job.id, root_id = %job_root, "updated parallel monitor lock for {uuid}");
}
}
let first_uuid = uuids[0];
let new_status = match next_status {
NextStatus::NextLoopIteration {
@@ -2610,36 +2705,52 @@ async fn push_next_flow_job<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
}
};
tracing::debug!("STATUS STEP: {:?} {i} {:#?}", status.step, new_status);
tracing::debug!("STATUS STEP: {:?} {step:?} {:#?}", status.step, new_status);
if i >= flow.modules.len() {
sqlx::query!(
"UPDATE queue
SET flow_status = JSONB_SET(
JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)
WHERE id = $3",
json!(FlowStatusModuleWParent {
parent_module: Some(current_id.clone()),
module_status: new_status
}),
json!(i),
flow_job.id
)
.execute(&mut tx)
.await?;
} else {
sqlx::query!(
"UPDATE queue
SET flow_status = JSONB_SET(
JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)
WHERE id = $4",
i as i32,
json!(new_status),
json!(i),
flow_job.id
)
.execute(&mut tx)
.await?;
match step {
Step::FailureStep => {
sqlx::query!(
"UPDATE queue
SET flow_status = JSONB_SET(
JSONB_SET(flow_status, ARRAY['failure_module'], $1), ARRAY['step'], $2)
WHERE id = $3",
json!(FlowStatusModuleWParent {
parent_module: Some(current_id.clone()),
module_status: new_status
}),
json!(flow.modules.len()),
flow_job.id
)
.execute(&mut tx)
.await?;
}
Step::PreprocessorStep => {
sqlx::query!(
"UPDATE queue
SET flow_status = JSONB_SET(
JSONB_SET(flow_status, ARRAY['preprocessor_module'], $1), ARRAY['step'], $2)
WHERE id = $3",
json!(new_status),
json!(-1),
flow_job.id
)
.execute(&mut tx)
.await?;
}
Step::Step(i) => {
sqlx::query!(
"UPDATE queue
SET flow_status = JSONB_SET(
JSONB_SET(flow_status, ARRAY['modules', $1::TEXT], $2), ARRAY['step'], $3)
WHERE id = $4",
i as i32,
json!(new_status),
json!(i),
flow_job.id
)
.execute(&mut tx)
.await?;
}
};
potentially_crash_for_testing();
@@ -2882,9 +2993,17 @@ async fn compute_next_flow_transform(
concurrency_time_window_s,
..
} => {
let path = path
.clone()
.or_else(|| Some(format!("{}/step-{}", flow_job.script_path(), status.step)));
let path = path.clone().or_else(|| {
if status
.preprocessor_module
.as_ref()
.is_some_and(|x| x.id() == module.id)
{
Some(format!("{}/preprocessor", flow_job.script_path()))
} else {
Some(format!("{}/step-{}", flow_job.script_path(), status.step))
}
});
let payload = raw_script_to_payload(
path,
content,
@@ -3011,6 +3130,7 @@ async fn compute_next_flow_transform(
priority: None,
early_return: None,
concurrency_key: None,
preprocessor_module: None,
};
ContinuePayload::ForloopJobs {
n: itered.len(),
@@ -3108,6 +3228,7 @@ async fn compute_next_flow_transform(
priority: None,
early_return: None,
concurrency_key: None,
preprocessor_module: None,
},
path: Some(format!(
"{}/branchone-{}",
@@ -3160,6 +3281,7 @@ async fn compute_next_flow_transform(
priority: None,
early_return: None,
concurrency_key: None,
preprocessor_module: None,
},
path: Some(format!(
"{}/branchall-{}",
@@ -3234,6 +3356,7 @@ async fn compute_next_flow_transform(
priority: None,
early_return: None,
concurrency_key: None,
preprocessor_module: None,
},
path: Some(format!(
"{}/branchall-{}",
@@ -3311,6 +3434,7 @@ async fn next_loop_iteration(
priority: None,
early_return: None,
concurrency_key: None,
preprocessor_module: None,
},
path: inner_path,
restarted_from: None,
@@ -3548,7 +3672,11 @@ fn raw_script_to_payload(
}
fn flow_to_payload(path: &str, delete_after_use: &bool) -> JobPayloadWithTag {
let payload = JobPayload::Flow { path: path.to_string(), dedicated_worker: None };
let payload = JobPayload::Flow {
path: path.to_string(),
dedicated_worker: None,
apply_preprocessor: false,
};
JobPayloadWithTag { payload, tag: None, delete_after_use: *delete_after_use, timeout: None }
}
@@ -3562,7 +3690,7 @@ async fn script_to_payload(
) -> Result<JobPayloadWithTag, Error> {
let (payload, tag, delete_after_use, script_timeout) = if script_hash.is_none() {
let (jp, tag, delete_after_use, script_timeout) =
script_path_to_payload(script_path, db, &flow_job.workspace_id).await?;
script_path_to_payload(script_path, db, &flow_job.workspace_id, Some(true)).await?;
(
jp,
tag_override.to_owned().or(tag),
@@ -3595,6 +3723,7 @@ async fn script_to_payload(
language,
dedicated_worker,
priority,
apply_preprocessor: false,
},
tag_override.to_owned().or(tag),
delete_after_use,
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.397.1";
export const VERSION = "v1.398.1";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/main.ts
+1 -1
View File
@@ -56,7 +56,7 @@ export {
// }
// });
export const VERSION = "1.397.1";
export const VERSION = "1.398.1";
const command = new Command()
.name("wmill")
cli/metadata.ts
+14 -4
View File
@@ -299,13 +299,15 @@ export async function updateScriptSchema(
): Promise<void> {
// infer schema from script content and update it inplace
await instantiateWasm();
const newSchema = inferSchema(
const result = inferSchema(
language,
scriptContent,
metadataContent.schema,
path
);
metadataContent.schema = newSchema;
metadataContent.schema = result.schema;
metadataContent.has_preprocessor = result.has_preprocessor;
metadataContent.no_main_func = result.no_main_func;
}
async function updateScriptLock(
@@ -489,7 +491,11 @@ export function inferSchema(
`Script ${path} invalid, it cannot be parsed to infer schema.`
)
);
return defaultScriptMetadata().schema;
return {
schema: defaultScriptMetadata().schema,
has_preprocessor: false,
no_main_func: false,
};
}
currentSchema.required = [];
@@ -515,7 +521,11 @@ export function inferSchema(
}
}
return currentSchema;
return {
schema: currentSchema,
has_preprocessor: inferedSchema.has_preprocessor,
no_main_func: inferedSchema.no_main_func,
};
}
function sortObject(obj: any): any {
cli/script.ts
+4 -4
View File
@@ -97,7 +97,6 @@ async function push(opts: PushOptions, filePath: string) {
}
export async function findResourceFile(path: string) {
const splitPath = path.split(".");
const contentBasePathJSON = splitPath[0] + "." + splitPath[1] + ".json";
@@ -124,9 +123,7 @@ export async function findResourceFile(path: string) {
);
}
if (validCandidates.length < 1) {
throw new Error(
`No resource matching file resource: ${path}.`
);
throw new Error(`No resource matching file resource: ${path}.`);
}
return validCandidates[0];
}
@@ -306,6 +303,7 @@ export async function handleFile(
restart_unless_cancelled: typed?.restart_unless_cancelled,
visible_to_runner_only: typed?.visible_to_runner_only,
no_main_func: typed?.no_main_func,
has_preprocessor: typed?.has_preprocessor,
priority: typed?.priority,
concurrency_key: typed?.concurrency_key,
//@ts-ignore
@@ -339,6 +337,8 @@ export async function handleFile(
Boolean(typed.visible_to_runner_only) ==
Boolean(remote.visible_to_runner_only) &&
Boolean(typed.no_main_func) == Boolean(remote.no_main_func) &&
Boolean(typed.has_preprocessor) ==
Boolean(remote.has_preprocessor) &&
typed.priority == Boolean(remote.priority) &&
typed.timeout == remote.timeout &&
//@ts-ignore
cli/sync.ts
+45 -26
View File
@@ -42,11 +42,8 @@ import {
generateFlowLockInternal,
generateScriptMetadataInternal,
} from "./metadata.ts";
<<<<<<< HEAD
import { pushResource } from "./resource.ts";
=======
import { FlowModule, OpenFlow, RawScript } from "./gen/types.gen.ts";
>>>>>>> origin/main
import { pushResource } from "./resource.ts";
type DynFSElement = {
isDirectory: boolean;
@@ -342,13 +339,15 @@ function ZipFSElement(
zip: JSZip,
useYaml: boolean,
defaultTs: "bun" | "deno",
resourceTypeToFormatExtension: Map<string, string>,
resourceTypeToFormatExtension: Record<string, string>
): DynFSElement {
async function _internal_file(
p: string,
f: JSZip.JSZipObject
): Promise<DynFSElement[]> {
const kind: "flow" | "app" | "script" | "resource" | "other" = p.endsWith("flow.json")
const kind: "flow" | "app" | "script" | "resource" | "other" = p.endsWith(
"flow.json"
)
? "flow"
: p.endsWith("app.json")
? "app"
@@ -457,16 +456,18 @@ function ZipFSElement(
: JSON.stringify(parsed, null, 2);
}
if (kind == 'resource') {
if (kind == "resource") {
const content = await f.async("text");
const parsed = JSON.parse(content);
const formatExtension = resourceTypeToFormatExtension[parsed["resource_type"]]
const formatExtension =
resourceTypeToFormatExtension[parsed["resource_type"]];
if (formatExtension) {
parsed["value"]["content"] = "!inline "
+ removeSuffix(p.replaceAll(SEP, "/"), ".resource.json")
+ ".resource.file." + formatExtension
parsed["value"]["content"] =
"!inline " +
removeSuffix(p.replaceAll(SEP, "/"), ".resource.json") +
".resource.file." +
formatExtension;
}
return useYaml
? yamlStringify(parsed, yamlOptions)
@@ -498,13 +499,17 @@ function ZipFSElement(
if (kind == "resource") {
const content = await f.async("text");
const parsed = JSON.parse(content);
const formatExtension = resourceTypeToFormatExtension[parsed["resource_type"]]
const formatExtension =
resourceTypeToFormatExtension[parsed["resource_type"]];
if (formatExtension) {
const fileContent: string = parsed["value"]["content"];
r.push({
isDirectory: false,
path: removeSuffix(finalPath, ".resource.json") + ".resource.file." + formatExtension,
path:
removeSuffix(finalPath, ".resource.json") +
".resource.file." +
formatExtension,
async *getChildren() {},
// deno-lint-ignore require-await
async getContentText() {
@@ -895,8 +900,14 @@ export async function pull(opts: GlobalOptions & SyncOptions) {
)
);
const resourceTypeToFormatExtension =
await wmill.fileResourceTypeToFileExtMap({ workspace: workspace.workspaceId })
let resourceTypeToFormatExtension: Record<string, string> = {};
try {
resourceTypeToFormatExtension = (await wmill.fileResourceTypeToFileExtMap({
workspace: workspace.workspaceId,
})) as Record<string, string>;
} catch {
// ignore
}
const remote = ZipFSElement(
(await downloadZip(
workspace,
@@ -913,7 +924,7 @@ export async function pull(opts: GlobalOptions & SyncOptions) {
))!,
!opts.json,
opts.defaultTs ?? "bun",
resourceTypeToFormatExtension,
resourceTypeToFormatExtension
);
const local = !opts.stateful
? await FSFSElement(Deno.cwd(), codebases)
@@ -1188,8 +1199,14 @@ export async function push(opts: GlobalOptions & SyncOptions) {
"Computing the files to update on the remote to match local (taking wmill.yaml includes/excludes into account)"
)
);
const resourceTypeToFormatExtension =
await wmill.fileResourceTypeToFileExtMap({ workspace: workspace.workspaceId })
let resourceTypeToFormatExtension: Record<string, string> = {};
try {
resourceTypeToFormatExtension = (await wmill.fileResourceTypeToFileExtMap({
workspace: workspace.workspaceId,
})) as Record<string, string>;
} catch {
// ignore
}
const remote = ZipFSElement(
(await downloadZip(
workspace,
@@ -1206,7 +1223,7 @@ export async function push(opts: GlobalOptions & SyncOptions) {
))!,
!opts.json,
opts.defaultTs ?? "bun",
resourceTypeToFormatExtension,
resourceTypeToFormatExtension
);
const local = await FSFSElement(path.join(Deno.cwd(), ""), codebases);
@@ -1291,24 +1308,26 @@ export async function push(opts: GlobalOptions & SyncOptions) {
}
if (isFileResource(change.path)) {
const resourceFilePath = await findResourceFile(change.path)
const resourceFilePath = await findResourceFile(change.path);
if (!alreadySynced.includes(resourceFilePath)) {
alreadySynced.push(resourceFilePath);
const newObj = parseFromPath(resourceFilePath, await Deno.readTextFile(resourceFilePath));
const newObj = parseFromPath(
resourceFilePath,
await Deno.readTextFile(resourceFilePath)
);
await pushResource(
workspace.workspaceId,
resourceFilePath,
undefined,
newObj,
newObj
);
if (opts.stateful && stateExists) {
await Deno.writeTextFile(stateTarget, change.after);
}
continue;
}
}
const oldObj = parseFromPath(change.path, change.before);
const newObj = parseFromPath(change.path, change.after);
@@ -1320,7 +1339,7 @@ export async function push(opts: GlobalOptions & SyncOptions) {
newObj,
opts.plainSecrets ?? false,
alreadySynced,
opts.message,
opts.message
);
if (opts.stateful && stateExists) {
@@ -1359,7 +1378,7 @@ export async function push(opts: GlobalOptions & SyncOptions) {
obj,
opts.plainSecrets ?? false,
[],
opts.message,
opts.message
);
if (opts.stateful && stateExists) {
cli/utils.ts
+6 -2
View File
@@ -3,6 +3,7 @@
// @ts-nocheck This file is copied from a JS project, so it's not type-safe.
import { log, encodeHex, SEP } from "./deps.ts";
import crypto from "node:crypto";
export function deepEqual<T>(a: T, b: T): boolean {
if (a === b) return true;
@@ -132,6 +133,9 @@ export function sleep(ms: number) {
export function isFileResource(path: string): boolean {
const splitPath = path.split(".");
return splitPath.length >= 4 && splitPath[1] == "resource" && splitPath[2] == "file";
return (
splitPath.length >= 4 &&
splitPath[1] == "resource" &&
splitPath[2] == "file"
);
}
cli/wasm/windmill_parser_wasm.generated.d.ts
Vendored
+11 -3
View File
@@ -19,7 +19,8 @@ export interface InstantiateResult {
parse_db_resource: typeof parse_db_resource;
parse_graphql: typeof parse_graphql;
parse_php: typeof parse_php;
parse_rust: typeof parse_rust
parse_rust: typeof parse_rust;
parse_ansible: typeof parse_ansible
};
}
@@ -46,9 +47,10 @@ export function instantiateWithInstance(opts?: InstantiateOptions): Promise<Inst
/**
* @param {string} code
* @param {string | undefined} [main_override]
* @returns {string}
*/
export function parse_deno(code: string): string;
export function parse_deno(code: string, main_override?: string): string;
/**
* @param {string} code
* @returns {string}
@@ -76,9 +78,10 @@ export function parse_powershell(code: string): string;
export function parse_go(code: string): string;
/**
* @param {string} code
* @param {string | undefined} [main_override]
* @returns {string}
*/
export function parse_python(code: string): string;
export function parse_python(code: string, main_override?: string): string;
/**
* @param {string} code
* @returns {string}
@@ -124,3 +127,8 @@ export function parse_php(code: string): string;
* @returns {string}
*/
export function parse_rust(code: string): string;
/**
* @param {string} code
* @returns {string}
*/
export function parse_ansible(code: string): string;
cli/wasm/windmill_parser_wasm.generated.js
+100 -54
View File
@@ -4,7 +4,7 @@
// deno-fmt-ignore-file
/// <reference types="./windmill_parser_wasm.generated.d.ts" />
// source-hash: d36b5583d8f6223c130a246a8713b8621d55b990
// source-hash: 6da62ad21f79c5b8fe442f1de939e367943e2665
let wasm;
const heap = new Array(128).fill(undefined);
@@ -15,6 +15,20 @@ function getObject(idx) {
return heap[idx];
}
let heap_next = heap.length;
function dropObject(idx) {
if (idx < 132) return;
heap[idx] = heap_next;
heap_next = idx;
}
function takeObject(idx) {
const ret = getObject(idx);
dropObject(idx);
return ret;
}
let WASM_VECTOR_LEN = 0;
let cachedUint8Memory0 = null;
@@ -89,38 +103,6 @@ function getInt32Memory0() {
return cachedInt32Memory0;
}
let heap_next = heap.length;
function dropObject(idx) {
if (idx < 132) return;
heap[idx] = heap_next;
heap_next = idx;
}
function takeObject(idx) {
const ret = getObject(idx);
dropObject(idx);
return ret;
}
let cachedFloat64Memory0 = null;
function getFloat64Memory0() {
if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) {
cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer);
}
return cachedFloat64Memory0;
}
function addHeapObject(obj) {
if (heap_next === heap.length) heap.push(heap.length + 1);
const idx = heap_next;
heap_next = heap[idx];
heap[idx] = obj;
return idx;
}
const cachedTextDecoder = typeof TextDecoder !== "undefined"
? new TextDecoder("utf-8", { ignoreBOM: true, fatal: true })
: {
@@ -136,6 +118,24 @@ function getStringFromWasm0(ptr, len) {
return cachedTextDecoder.decode(getUint8Memory0().subarray(ptr, ptr + len));
}
function addHeapObject(obj) {
if (heap_next === heap.length) heap.push(heap.length + 1);
const idx = heap_next;
heap_next = heap[idx];
heap[idx] = obj;
return idx;
}
let cachedFloat64Memory0 = null;
function getFloat64Memory0() {
if (cachedFloat64Memory0 === null || cachedFloat64Memory0.byteLength === 0) {
cachedFloat64Memory0 = new Float64Array(wasm.memory.buffer);
}
return cachedFloat64Memory0;
}
let cachedBigInt64Memory0 = null;
function getBigInt64Memory0() {
@@ -213,11 +213,12 @@ function debugString(val) {
}
/**
* @param {string} code
* @param {string | undefined} [main_override]
* @returns {string}
*/
export function parse_deno(code) {
let deferred2_0;
let deferred2_1;
export function parse_deno(code, main_override) {
let deferred3_0;
let deferred3_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(
@@ -226,15 +227,23 @@ export function parse_deno(code) {
wasm.__wbindgen_realloc,
);
const len0 = WASM_VECTOR_LEN;
wasm.parse_deno(retptr, ptr0, len0);
var ptr1 = isLikeNone(main_override)
? 0
: passStringToWasm0(
main_override,
wasm.__wbindgen_malloc,
wasm.__wbindgen_realloc,
);
var len1 = WASM_VECTOR_LEN;
wasm.parse_deno(retptr, ptr0, len0, ptr1, len1);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
deferred3_0 = r0;
deferred3_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
wasm.__wbindgen_free(deferred3_0, deferred3_1, 1);
}
}
@@ -375,11 +384,12 @@ export function parse_go(code) {
/**
* @param {string} code
* @param {string | undefined} [main_override]
* @returns {string}
*/
export function parse_python(code) {
let deferred2_0;
let deferred2_1;
export function parse_python(code, main_override) {
let deferred3_0;
let deferred3_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(
@@ -388,15 +398,23 @@ export function parse_python(code) {
wasm.__wbindgen_realloc,
);
const len0 = WASM_VECTOR_LEN;
wasm.parse_python(retptr, ptr0, len0);
var ptr1 = isLikeNone(main_override)
? 0
: passStringToWasm0(
main_override,
wasm.__wbindgen_malloc,
wasm.__wbindgen_realloc,
);
var len1 = WASM_VECTOR_LEN;
wasm.parse_python(retptr, ptr0, len0, ptr1, len1);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
deferred3_0 = r0;
deferred3_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
wasm.__wbindgen_free(deferred3_0, deferred3_1, 1);
}
}
@@ -643,6 +661,33 @@ export function parse_rust(code) {
}
}
/**
* @param {string} code
* @returns {string}
*/
export function parse_ansible(code) {
let deferred2_0;
let deferred2_1;
try {
const retptr = wasm.__wbindgen_add_to_stack_pointer(-16);
const ptr0 = passStringToWasm0(
code,
wasm.__wbindgen_malloc,
wasm.__wbindgen_realloc,
);
const len0 = WASM_VECTOR_LEN;
wasm.parse_ansible(retptr, ptr0, len0);
var r0 = getInt32Memory0()[retptr / 4 + 0];
var r1 = getInt32Memory0()[retptr / 4 + 1];
deferred2_0 = r0;
deferred2_1 = r1;
return getStringFromWasm0(r0, r1);
} finally {
wasm.__wbindgen_add_to_stack_pointer(16);
wasm.__wbindgen_free(deferred2_0, deferred2_1, 1);
}
}
function handleError(f, args) {
try {
return f.apply(this, args);
@@ -653,6 +698,9 @@ function handleError(f, args) {
const imports = {
__wbindgen_placeholder__: {
__wbindgen_object_drop_ref: function (arg0) {
takeObject(arg0);
},
__wbindgen_string_get: function (arg0, arg1) {
const obj = getObject(arg1);
const ret = typeof obj === "string" ? obj : undefined;
@@ -667,8 +715,9 @@ const imports = {
getInt32Memory0()[arg0 / 4 + 1] = len1;
getInt32Memory0()[arg0 / 4 + 0] = ptr1;
},
__wbindgen_object_drop_ref: function (arg0) {
takeObject(arg0);
__wbindgen_error_new: function (arg0, arg1) {
const ret = new Error(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
},
__wbindgen_boolean_get: function (arg0) {
const v = getObject(arg0);
@@ -706,11 +755,7 @@ const imports = {
const ret = BigInt.asUintN(64, arg0);
return addHeapObject(ret);
},
__wbindgen_error_new: function (arg0, arg1) {
const ret = new Error(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
},
__wbg_eval_c227cc6614cd76e5: function (arg0, arg1) {
__wbg_eval_b7405fdd08ddef3d: function (arg0, arg1) {
const ret = eval(getStringFromWasm0(arg0, arg1));
return addHeapObject(ret);
},
@@ -993,6 +1038,7 @@ function getWasmInstanceExports() {
parse_graphql,
parse_php,
parse_rust,
parse_ansible,
};
}
cli/wasm/windmill_parser_wasm_bg.wasm
BIN
View File
Binary file not shown.
docker/DockerfileSlim
+3 -1
View File
@@ -1,7 +1,9 @@
ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm
ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm
FROM ${PYTHON_IMAGE}
RUN pip install --upgrade pip==24.2
ARG APP=/usr/src/app
RUN apt-get update \
docker/DockerfileSlimEe
+3 -1
View File
@@ -1,7 +1,9 @@
ARG PYTHON_IMAGE=python:3.11.8-slim-bookworm
ARG PYTHON_IMAGE=python:3.11.10-slim-bookworm
FROM ${PYTHON_IMAGE}
RUN pip install --upgrade pip==24.2
ARG APP=/usr/src/app
RUN apt-get update \

Some files were not shown because too many files have changed in this diff Show More