starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-05 00:03:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fix-no-uv-fallback

Browse Source
This commit is contained in:
pyranota
2024-12-09 18:32:14 +01:00
committed by GitHub
parent 484a612017 2c5d07f3d9
commit 1106f72daa
168 changed files with 6084 additions and 2847 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/change-versions-mac.sh
+1 -1
View File
@@ -16,7 +16,7 @@ sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/
sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i '' -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
sed -i '' -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
# sed -i '' -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
sed -i '' -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
sed -i '' -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
.github/change-versions.sh
+1 -1
View File
@@ -17,7 +17,7 @@ sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/pac
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
sed -i -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
# sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
.github/workflows/docker-image.yml
+3 -3
View File
@@ -76,7 +76,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,jemalloc,deno_core
features=embedding,parquet,openidconnect,jemalloc,deno_core,dind
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
${{ steps.meta-public.outputs.tags }}
@@ -138,7 +138,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
${{ steps.meta-ee-public.outputs.tags }}
@@ -200,7 +200,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind
PYTHON_IMAGE=python:3.12.2-slim-bookworm
tags: |
${{ steps.meta-ee-public-py312.outputs.tags }}
CHANGELOG.md
+26
View File
@@ -1,5 +1,31 @@
# Changelog
## [1.435.2](https://github.com/windmill-labs/windmill/compare/v1.435.1...v1.435.2) (2024-12-05)
### Bug Fixes
* job search toast on error ([#4851](https://github.com/windmill-labs/windmill/issues/4851)) ([a99e63f](https://github.com/windmill-labs/windmill/commit/a99e63f5435725c42e38b46933ff3caa345002b5))
## [1.435.1](https://github.com/windmill-labs/windmill/compare/v1.435.0...v1.435.1) (2024-12-05)
### Bug Fixes
* improve critical alerts filters ([548cfcf](https://github.com/windmill-labs/windmill/commit/548cfcfbde23ac7ae129e10f6e92d57516e61f20))
## [1.435.0](https://github.com/windmill-labs/windmill/compare/v1.434.2...v1.435.0) (2024-12-05)
### Features
* app custom paths ([#4828](https://github.com/windmill-labs/windmill/issues/4828)) ([1ec6c6f](https://github.com/windmill-labs/windmill/commit/1ec6c6f765904361e641d89495890bc87e8544aa))
### Bug Fixes
* pass USERPROFILE on windows ([5404ec9](https://github.com/windmill-labs/windmill/commit/5404ec9b48e8d7a0cb27b2319d54f04c94ec6fd0))
## [1.434.2](https://github.com/windmill-labs/windmill/compare/v1.434.1...v1.434.2) (2024-12-04)
Dockerfile
+2 -2
View File
@@ -175,9 +175,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
COPY --from=builder /frontend/build /static_frontend
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=denoland/deno:2.0.4 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=denoland/deno:2.1.2 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
backend/.sqlx/query-0a46f1f3047d15227f82ae24ad2113eb91d65b98927eaaba427cbde27dd79bfe.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*)\n FROM alerts\n WHERE COALESCE(acknowledged, false) = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Bool"
]
},
"nullable": [
null
]
},
"hash": "0a46f1f3047d15227f82ae24ad2113eb91d65b98927eaaba427cbde27dd79bfe"
}
backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json
Generated
+2 -1
View File
@@ -44,7 +44,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO app_script (app, hash, lock, code, code_sha256)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8",
"Bpchar",
"Text",
"Text",
"Bpchar"
]
},
"nullable": [
false
]
},
"hash": "0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b"
}
backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT format('rawscript/%s', code_sha256) as \"path!: String\"\n FROM app_script WHERE id = $1 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path!: String",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
null
]
},
"hash": "1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013"
}
backend/.sqlx/query-1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20"
}
backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json
Generated
+2 -1
View File
@@ -34,7 +34,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-372eb162ace15a4f07162bb46706eaca91a3616c9bfbd78a408b7079ca9706d4.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*)\n FROM alerts\n WHERE workspace_id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "372eb162ace15a4f07162bb46706eaca91a3616c9bfbd78a408b7079ca9706d4"
}
backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json
Generated
+27
View File
@@ -0,0 +1,27 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bpchar",
"Text",
"Text",
"Jsonb"
]
},
"nullable": [
false
]
},
"hash": "5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d"
}
backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT policy as \"policy: sqlx::types::Json<Box<RawValue>>\"\n FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "policy: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c"
}
backend/.sqlx/query-65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts\n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true\n ELSE acknowledged_workspace\n END\n WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int4",
"Text"
]
},
"nullable": []
},
"hash": "65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a"
}
backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json → backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at",
"query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at, worker_group FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at",
"describe": {
"columns": [
{
@@ -32,6 +32,11 @@
"ordinal": 5,
"name": "started_at",
"type_info": "Timestamptz"
},
{
"ordinal": 6,
"name": "worker_group",
"type_info": "Varchar"
}
],
"parameters": {
@@ -43,8 +48,9 @@
true,
true,
false,
false,
false
]
},
"hash": "a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7"
"hash": "6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973"
}
backend/.sqlx/query-75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2.json → backend/.sqlx/query-6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5.json
Generated
+4 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO app\n (workspace_id, path, summary, policy, versions, draft_only)\n VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id",
"query": "INSERT INTO app\n (workspace_id, path, summary, policy, versions, draft_only, custom_path)\n VALUES ($1, $2, $3, $4, '{}', $5, $6) RETURNING id",
"describe": {
"columns": [
{
@@ -15,12 +15,13 @@
"Varchar",
"Varchar",
"Jsonb",
"Bool"
"Bool",
"Text"
]
},
"nullable": [
false
]
},
"hash": "75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2"
"hash": "6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5"
}
backend/.sqlx/query-7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3.json
Generated
-14
View File
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts \n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $1::text IS NOT NULL THEN true\n ELSE acknowledged_workspace\n END\n WHERE ($1::text IS NOT NULL AND workspace_id = $1)\n OR ($1::text IS NULL)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text"
]
},
"nullable": []
},
"hash": "7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3"
}
backend/.sqlx/query-80864158f61adaad8df934acc54ba523c9f17d106298d8781885134d28553d36.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts \n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $2 THEN\n CASE\n WHEN $1::text IS NOT NULL THEN true\n ELSE acknowledged_workspace\n END\n ELSE true\n END\n WHERE ($1::text IS NOT NULL AND workspace_id = $1)\n OR ($1::text IS NULL)",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "80864158f61adaad8df934acc54ba523c9f17d106298d8781885134d28553d36"
}
backend/.sqlx/query-83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806.json
Generated
+27
View File
@@ -0,0 +1,27 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bpchar",
"Text",
"Text",
"Jsonb"
]
},
"nullable": [
false
]
},
"hash": "83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806"
}
backend/.sqlx/query-951c053cf756f00e1da26b06edb3d0193a0ee707e6482a892db58915e0c8a27f.json
Generated
+20
View File
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*)\n FROM alerts",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "951c053cf756f00e1da26b06edb3d0193a0ee707e6482a892db58915e0c8a27f"
}
backend/.sqlx/query-b9ed42d4b795942251baafd016b4361e75257c0e5faf8795274ec86236a27413.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*)\n FROM alerts\n WHERE workspace_id = $1 AND COALESCE(acknowledged_workspace, false) = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "count",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Text",
"Bool"
]
},
"nullable": [
null
]
},
"hash": "b9ed42d4b795942251baafd016b4361e75257c0e5faf8795274ec86236a27413"
}
backend/.sqlx/query-cd4067e68b375461a495f402d05976da6c6e331d5748bf6f8d59f9f75c027fe8.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE alerts\n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $3 THEN\n CASE\n WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true\n ELSE acknowledged_workspace\n END\n ELSE true\n END\n WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int4",
"Text",
"Bool"
]
},
"nullable": []
},
"hash": "cd4067e68b375461a495f402d05976da6c6e331d5748bf6f8d59f9f75c027fe8"
}
backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT app_id, value FROM app_version WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "app_id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Json"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false,
false
]
},
"hash": "ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b"
}
backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO app_version_lite (id, value) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Jsonb"
]
},
"nullable": []
},
"hash": "ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602"
}
backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json
Generated
+2 -1
View File
@@ -48,7 +48,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df.json
Generated
+25
View File
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text",
"Text"
]
},
"nullable": [
null
]
},
"hash": "fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df"
}
backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "lock",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "code",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
true,
false
]
},
"hash": "ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c"
}
backend/Cargo.lock
Generated
+418 -119
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+27 -13
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.434.2"
version = "1.435.2"
authors.workspace = true
edition.workspace = true
@@ -29,7 +29,7 @@ members = [
]
[workspace.package]
version = "1.434.2"
version = "1.435.2"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -63,6 +63,8 @@ tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"]
sqlx = ["windmill-worker/sqlx"]
deno_core = ["windmill-worker/deno_core", "dep:deno_core"]
kafka = ["windmill-api/kafka"]
otel = ["windmill-common/otel", "windmill-worker/otel"]
dind = ["windmill-worker/dind"]
[dependencies]
anyhow.workspace = true
@@ -95,7 +97,6 @@ deno_core = { workspace = true, optional = true }
object_store = { workspace = true, optional = true }
quote.workspace = true
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { optional = true, workspace = true }
tikv-jemalloc-sys = { optional = true, workspace = true }
@@ -137,7 +138,7 @@ windmill-api-client = { path = "./windmill-api-client" }
axum = { version = "^0.7", features = ["multipart"] }
headers = "^0"
hyper = { version = "^1", features = ["full"] }
tokio = { version = "^1", features = ["full", "tracing"] }
tokio = { version = "^1.42.0", features = ["full", "tracing"] }
tower = "^0"
tower-http = { version = "^0.6", features = ["trace", "cors"] }
tower-cookies = "^0.10"
@@ -182,15 +183,16 @@ itertools = "^0"
regex = "^1"
semver = "^1"
deno_fetch = "0.195.0"
deno_tls = "0.158.0"
deno_console = "0.171.0"
deno_url = "0.171.0"
deno_webidl = "0.171.0"
deno_web = "0.202.0"
deno_net = "0.163.0"
deno_core = "0.311.0"
deno_ast = { version = "=0.42.2", features = ["transpiling"] }
deno_fetch = "0.203.0"
deno_tls = "0.166.0"
deno_console = "0.179.0"
deno_url = "0.179.0"
deno_webidl = "0.179.0"
deno_web = "0.210.0"
deno_net = "0.171.0"
deno_core = "0.321.0"
deno_ast = { version = "=0.43.3", features = ["transpiling"] }
deno_permissions = "0.39.0"
swc_common = "=0.37.5"
swc_ecma_parser = "=0.149.1"
@@ -279,11 +281,23 @@ tar = "^0"
http = "^1"
async-stream = "^0"
opentelemetry = "0.27.0"
tracing-opentelemetry = "0.28.0"
opentelemetry_sdk = { version = "*", features = ["rt-tokio"] }
opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] }
opentelemetry-appender-tracing = "0.27.0"
opentelemetry-semantic-conventions = { version = "*", features = ["semconv_experimental"] }
bollard = "0.18.1"
tonic = { version = "^0", features = ["tls-native-roots"] }
tikv-jemallocator = { version = "0.5" }
tikv-jemalloc-sys = { version = "^0.5" }
tikv-jemalloc-ctl = { version = "^0.5" }
triomphe = "^0"
pin-project-lite = "^0"
tantivy = "0.22.0"
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
aefbc1e2188fea312996fcfc30a29d8fb5315316
a07bc62582c809457f1c945d6cda145770b94d04
backend/migrations/20241202095902_app_script.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add down migration script here
DROP TABLE IF EXISTS app_version_lite;
DROP TABLE IF EXISTS app_script;
backend/migrations/20241202095902_app_script.up.sql
+26
View File
@@ -0,0 +1,26 @@
-- Add up migration script here
ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'appscript';
-- Same as `app_version` but with a "lite" value (w/ `inlineScript.{code,lock}`).
CREATE TABLE app_version_lite (
id BIGSERIAL PRIMARY KEY,
value JSONB,
FOREIGN KEY (id) REFERENCES app_version (id) ON DELETE CASCADE
);
GRANT ALL ON app_version_lite TO windmill_user;
GRANT ALL ON app_version_lite TO windmill_admin;
-- App `inlineScript`.
CREATE TABLE app_script (
id BIGSERIAL PRIMARY KEY,
app BIGSERIAL NOT NULL,
hash CHAR(64) NOT NULL UNIQUE, -- sha256 of `app`, `lock`, `code`.
lock TEXT,
code TEXT NOT NULL,
code_sha256 CHAR(64) NOT NULL, -- used to retrieve the policy.
FOREIGN KEY (app) REFERENCES app (id) ON DELETE CASCADE
);
GRANT ALL ON app_script TO windmill_user;
GRANT ALL ON app_script TO windmill_admin;
backend/migrations/20241202134622_app_custom_path.down.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add down migration script here
ALTER TABLE app DROP COLUMN custom_path;
backend/migrations/20241202134622_app_custom_path.up.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TABLE app ADD COLUMN custom_path TEXT CHECK (custom_path ~ '^[\w-]+(\/[\w-]+)*$');
backend/migrations/20241204154025_grant_all_concurrency_key.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20241204154025_grant_all_concurrency_key.up.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add up migration script here
GRANT ALL ON concurrency_key TO windmill_admin;
GRANT ALL ON concurrency_key TO windmill_user;
backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add down migration script here
ALTER TABLE flow_node DROP COLUMN hash_v2;
backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql
+4
View File
@@ -0,0 +1,4 @@
-- Add up migration script here
CREATE SEQUENCE IF NOT EXISTS flow_node_hash_seq;
ALTER TABLE flow_node ALTER COLUMN hash DROP NOT NULL;
ALTER TABLE flow_node ADD COLUMN hash_v2 CHAR(64) NOT NULL UNIQUE DEFAULT to_hex(nextval('flow_node_hash_seq'));
backend/migrations/20241206075559_flow_node_unique_2.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add down migration script here
ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_unique_2;
ALTER TABLE flow_node ADD CONSTRAINT flow_node_hash_v2_key UNIQUE (hash_v2);
backend/migrations/20241206075559_flow_node_unique_2.up.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add up migration script here
ALTER TABLE flow_node ADD CONSTRAINT flow_node_unique_2 UNIQUE (path, workspace_id, hash_v2);
ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_hash_v2_key;
backend/src/main.rs
+90 -68
View File
@@ -8,7 +8,7 @@
use anyhow::Context;
use monitor::{
reload_delete_logs_periodically_setting, reload_indexer_config,
load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
send_logs_to_object_store,
};
@@ -37,7 +37,7 @@ use windmill_common::{
EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
},
@@ -221,11 +221,66 @@ async fn windmill_main() -> anyhow::Result<()> {
let hostname = hostname();
#[cfg(not(feature = "flamegraph"))]
let _guard = windmill_common::tracing_init::initialize_tracing(&hostname);
let mut enable_standalone_indexer: bool = false;
let mode = std::env::var("MODE")
.map(|x| x.to_lowercase())
.map(|x| {
if &x == "server" {
println!("Binary is in 'server' mode");
Mode::Server
} else if &x == "worker" {
tracing::info!("Binary is in 'worker' mode");
#[cfg(windows)]
{
println!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL");
}
Mode::Worker
} else if &x == "agent" {
println!("Binary is in 'agent' mode");
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
}
#[cfg(not(feature = "enterprise"))]
{
panic!("Agent mode is only available in the EE, ignoring...");
}
#[cfg(feature = "enterprise")]
Mode::Agent
} else if &x == "indexer" {
tracing::info!("Binary is in 'indexer' mode");
#[cfg(not(feature = "tantivy"))]
{
eprintln!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features.");
panic!("Indexer mode requires compiling with the tantivy feature flag.");
}
#[cfg(feature = "tantivy")]
Mode::Indexer
} else if &x == "standalone+search"{
enable_standalone_indexer = true;
println!("Binary is in 'standalone' mode with search enabled");
Mode::Standalone
}
else {
if &x != "standalone" {
eprintln!("mode not recognized, defaulting to standalone: {x}");
} else {
println!("Binary is in 'standalone' mode");
}
Mode::Standalone
}
})
.unwrap_or_else(|_| {
tracing::info!("Mode not specified, defaulting to standalone");
Mode::Standalone
});
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
tracing::info!("jemalloc enabled");
println!("jemalloc enabled");
#[cfg(feature = "flamegraph")]
let _guard = windmill_common::tracing_init::setup_flamegraph();
@@ -236,13 +291,13 @@ async fn windmill_main() -> anyhow::Result<()> {
"cache" => {
#[cfg(feature = "embedding")]
{
tracing::info!("Caching embedding model...");
println!("Caching embedding model...");
windmill_api::embeddings::ModelInstance::load_model_files().await?;
tracing::info!("Cached embedding model");
println!("Cached embedding model");
}
#[cfg(not(feature = "embedding"))]
{
tracing::warn!("Embeddings are not enabled, ignoring...");
println!("Embeddings are not enabled, ignoring...");
}
cache_hub_scripts(std::env::args().nth(2)).await?;
@@ -256,64 +311,6 @@ async fn windmill_main() -> anyhow::Result<()> {
_ => {}
}
let mut enable_standalone_indexer: bool = false;
let mode = std::env::var("MODE")
.map(|x| x.to_lowercase())
.map(|x| {
if &x == "server" {
tracing::info!("Binary is in 'server' mode");
Mode::Server
} else if &x == "worker" {
tracing::info!("Binary is in 'worker' mode");
#[cfg(windows)]
{
tracing::warn!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL");
}
Mode::Worker
} else if &x == "agent" {
tracing::info!("Binary is in 'agent' mode");
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
tracing::warn!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
}
#[cfg(not(feature = "enterprise"))]
{
panic!("Agent mode is only available in the EE, ignoring...");
}
#[cfg(feature = "enterprise")]
Mode::Agent
} else if &x == "indexer" {
tracing::info!("Binary is in 'indexer' mode");
#[cfg(not(feature = "tantivy"))]
{
tracing::error!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features.");
panic!("Indexer mode requires compiling with the tantivy feature flag.");
}
#[cfg(feature = "tantivy")]
Mode::Indexer
} else if &x == "standalone+search"{
enable_standalone_indexer = true;
tracing::info!("Binary is in 'standalone' mode with search enabled");
Mode::Standalone
}
else {
if &x != "standalone" {
tracing::error!("mode not recognized, defaulting to standalone: {x}");
} else {
tracing::info!("Binary is in 'standalone' mode");
}
Mode::Standalone
}
})
.unwrap_or_else(|_| {
tracing::info!("Mode not specified, defaulting to standalone");
Mode::Standalone
});
#[allow(unused_mut)]
let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer {
0
@@ -325,7 +322,7 @@ async fn windmill_main() -> anyhow::Result<()> {
};
if num_workers > 1 {
tracing::warn!(
println!(
"We STRONGLY recommend using at most 1 worker per container, use at your own risks"
);
}
@@ -347,10 +344,26 @@ async fn windmill_main() -> anyhow::Result<()> {
IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))
};
tracing::info!("Connecting to database...");
println!("Connecting to database...");
let db = windmill_common::connect_db(server_mode, indexer_mode).await?;
load_otel(&db).await;
tracing::info!("Database connected");
let environment = load_base_url(&db)
.await
.unwrap_or_else(|_| "local".to_string())
.trim_start_matches("https://")
.trim_start_matches("http://")
.split(".")
.next()
.unwrap_or_else(|| "local")
.to_string();
#[cfg(not(feature = "flamegraph"))]
let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment);
let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await;
tracing::info!(
@@ -776,6 +789,15 @@ Windmill Community Edition {GIT_VERSION}
tracing::error!(error = %e, "Could not reload debug metrics setting");
}
},
OTEL_SETTING => {
tracing::info!("OTEL setting changed, restarting");
// we wait a bit randomly to avoid having all servers and workers shutdown at same time
let rd_delay = rand::thread_rng().gen_range(0..4);
tokio::time::sleep(Duration::from_secs(rd_delay)).await;
if let Err(e) = tx.send(()) {
tracing::error!(error = %e, "Could not send killpill");
}
},
REQUEST_SIZE_LIMIT_SETTING => {
if server_mode {
tracing::info!("Request limit size change detected, killing server expecting to be restarted");
backend/src/monitor.rs
+75 -11
View File
@@ -12,7 +12,7 @@ use std::{
use chrono::{NaiveDateTime, Utc};
use futures::{stream::FuturesUnordered, StreamExt};
use serde::de::DeserializeOwned;
use serde::{de::DeserializeOwned, Deserializer};
use sqlx::{Pool, Postgres};
use tokio::{
join,
@@ -40,7 +40,7 @@ use windmill_common::{
EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
},
@@ -58,7 +58,8 @@ use windmill_common::{
},
BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
MONITOR_LOGS_ON_OBJECT_STORE, SERVICE_LOG_RETENTION_SECS,
MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
SERVICE_LOG_RETENTION_SECS,
};
use windmill_queue::cancel_job;
use windmill_worker::{
@@ -85,12 +86,12 @@ lazy_static::lazy_static! {
static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT")
.ok()
.and_then(|x| x.parse::<String>().ok())
.unwrap_or_else(|| "30".to_string());
.unwrap_or_else(|| "60".to_string());
static ref FLOW_ZOMBIE_TRANSITION_TIMEOUT: String = std::env::var("FLOW_ZOMBIE_TRANSITION_TIMEOUT")
.ok()
.and_then(|x| x.parse::<String>().ok())
.unwrap_or_else(|| "30".to_string());
.unwrap_or_else(|| "60".to_string());
pub static ref RESTART_ZOMBIE_JOBS: bool = std::env::var("RESTART_ZOMBIE_JOBS")
@@ -199,6 +200,65 @@ pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> {
Ok(())
}
fn empty_string_as_none<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
where
D: Deserializer<'de>,
{
let option = <Option<String> as serde::Deserialize>::deserialize(deserializer)?;
Ok(option.filter(|s| !s.is_empty()))
}
#[derive(serde::Deserialize)]
struct OtelSetting {
metrics_enabled: Option<bool>,
logs_enabled: Option<bool>,
tracing_enabled: Option<bool>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_endpoint: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_headers: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_protocol: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_compression: Option<String>,
}
pub async fn load_otel(db: &DB) {
let otel = load_value_from_global_settings(db, OTEL_SETTING).await;
if let Ok(v) = otel {
if let Some(v) = v {
let deser = serde_json::from_value::<OtelSetting>(v);
if let Ok(o) = deser {
let metrics_enabled = o.metrics_enabled.unwrap_or(false);
let logs_enabled = o.logs_enabled.unwrap_or(false);
let tracing_enabled = o.tracing_enabled.unwrap_or(false);
OTEL_METRICS_ENABLED.store(metrics_enabled, Ordering::Relaxed);
OTEL_LOGS_ENABLED.store(logs_enabled, Ordering::Relaxed);
OTEL_TRACING_ENABLED.store(tracing_enabled, Ordering::Relaxed);
if let Some(endpoint) = o.otel_exporter_otlp_endpoint.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
}
if let Some(headers) = o.otel_exporter_otlp_headers.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", headers);
}
if let Some(protocol) = o.otel_exporter_otlp_protocol {
std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", protocol);
}
if let Some(compression) = o.otel_exporter_otlp_compression {
std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", compression);
}
println!("OTEL settings loaded: tracing ({tracing_enabled}), logs ({logs_enabled}), metrics ({metrics_enabled}), endpoint ({:?}), headers defined: ({})",
o.otel_exporter_otlp_endpoint, o.otel_exporter_otlp_headers.is_some());
} else {
tracing::error!("Error deserializing otel settings");
}
}
} else {
tracing::error!("Error loading otel settings: {}", otel.unwrap_err());
}
}
pub async fn load_tag_per_workspace_enabled(db: &DB) -> error::Result<()> {
let metrics_enabled =
load_value_from_global_settings(db, DEFAULT_TAGS_PER_WORKSPACE_SETTING).await;
@@ -1337,7 +1397,7 @@ pub async fn reload_worker_config(
}
}
pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
pub async fn load_base_url(db: &DB) -> error::Result<String> {
let q_base_url = load_value_from_global_settings(db, BASE_URL_SETTING).await?;
let std_base_url = std::env::var("BASE_URL")
@@ -1361,6 +1421,14 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
std_base_url
};
{
let mut l = BASE_URL.write().await;
*l = base_url.clone();
}
Ok(base_url)
}
pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?;
let oauths = if let Some(q) = q_oauth {
@@ -1374,6 +1442,7 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
None
};
let base_url = load_base_url(db).await?;
let is_secure = base_url.starts_with("https://");
{
@@ -1383,11 +1452,6 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
.unwrap();
}
{
let mut l = BASE_URL.write().await;
*l = base_url
}
{
let mut l = IS_SECURE.write().await;
*l = is_secure;
backend/tests/worker.rs
+5 -1
View File
@@ -77,7 +77,11 @@ async fn initialize_tracing() {
static ONCE: Once = Once::new();
ONCE.call_once(|| {
let _ = windmill_common::tracing_init::initialize_tracing("test");
let _ = windmill_common::tracing_init::initialize_tracing(
"test",
&windmill_common::utils::Mode::Standalone,
"test",
);
});
}
backend/windmill-api/openapi-deref.yaml
+6
View File
@@ -8709,6 +8709,9 @@ paths:
- identity
- deploymentcallback
- singlescriptflow
- flowscript
- flownode
- appscript
schedule_path:
type: string
permissioned_as:
@@ -9267,6 +9270,9 @@ paths:
- identity
- deploymentcallback
- singlescriptflow
- flowscript
- flownode
- appscript
schedule_path:
type: string
permissioned_as:
backend/windmill-api/openapi.yaml
+104 -7
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.434.2
version: 1.435.2
title: Windmill API
contact:
@@ -872,9 +872,20 @@ paths:
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/CriticalAlert'
type: object
properties:
alerts:
type: array
items:
$ref: '#/components/schemas/CriticalAlert'
total_rows:
type: integer
description: Total number of rows matching the query.
example: 100
total_pages:
type: integer
description: Total number of pages based on the page size.
example: 10
/settings/critical_alerts/{id}/acknowledge:
post:
@@ -2757,9 +2768,20 @@ paths:
content:
application/json:
schema:
type: array
items:
$ref: '#/components/schemas/CriticalAlert'
type: object
properties:
alerts:
type: array
items:
$ref: '#/components/schemas/CriticalAlert'
total_rows:
type: integer
description: Total number of rows matching the query.
example: 100
total_pages:
type: integer
description: Total number of pages based on the page size.
example: 10
/w/{workspace}/workspaces/critical_alerts/{id}/acknowledge:
post:
@@ -3733,6 +3755,27 @@ paths:
required:
- app
/apps_u/public_app_by_custom_path/{custom_path}:
get:
summary: get public app by custom path
operationId: getPublicAppByCustomPath
tags:
- app
parameters:
- $ref: "#/components/parameters/CustomPath"
responses:
"200":
description: app details
content:
application/json:
schema:
allOf:
- $ref: "#/components/schemas/AppWithLastVersion"
- type: object
properties:
workspace_id:
type: string
/scripts/hub/get/{path}:
get:
summary: get hub script content by path
@@ -5371,6 +5414,8 @@ paths:
type: boolean
deployment_message:
type: string
custom_path:
type: string
required:
- path
- value
@@ -5422,6 +5467,23 @@ paths:
schema:
$ref: "#/components/schemas/AppWithLastVersion"
/w/{workspace}/apps/get/lite/{path}:
get:
summary: get app lite by path
operationId: getAppLiteByPath
tags:
- app
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/ScriptPath"
responses:
"200":
description: app lite details
content:
application/json:
schema:
$ref: "#/components/schemas/AppWithLastVersion"
/w/{workspace}/apps/get/draft/{path}:
get:
summary: get app by path with draft
@@ -5696,6 +5758,8 @@ paths:
$ref: "#/components/schemas/Policy"
deployment_message:
type: string
custom_path:
type: string
responses:
"200":
description: app updated
@@ -5704,6 +5768,23 @@ paths:
schema:
type: string
/w/{workspace}/apps/custom_path_exists/{custom_path}:
get:
summary: check if custom path exists
operationId: customPathExists
tags:
- app
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/CustomPath"
responses:
"200":
description: custom path exists
content:
application/json:
schema:
type: boolean
/w/{workspace}/apps_u/execute_component/{path}:
post:
summary: executeComponent
@@ -5727,6 +5808,8 @@ paths:
#flow: flow/<path>
path:
type: string
version:
type: integer
args: {}
raw_code:
type: object
@@ -5744,6 +5827,8 @@ paths:
required:
- content
- language
id:
type: integer
force_viewer_static_fields:
type: object
force_viewer_one_of_fields:
@@ -10222,6 +10307,12 @@ components:
required: true
schema:
type: string
CustomPath:
name: custom_path
in: path
required: true
schema:
type: string
PathId:
name: id
in: path
@@ -10892,6 +10983,8 @@ components:
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
]
schedule_path:
type: string
@@ -11012,6 +11105,8 @@ components:
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
]
schedule_path:
type: string
@@ -12860,6 +12955,8 @@ components:
draft_only:
type: boolean
draft: {}
custom_path:
type: string
AppHistory:
type: object
backend/windmill-api/src/apps.rs
+274 -153
View File
@@ -1,4 +1,4 @@
use std::collections::HashMap;
use std::{collections::HashMap, sync::Arc};
/*
* Author: Ruben Fiszel
@@ -31,6 +31,7 @@ use axum::{
routing::{delete, get, post},
Router,
};
use futures::future::{FutureExt, TryFutureExt};
use hyper::StatusCode;
#[cfg(feature = "parquet")]
use itertools::Itertools;
@@ -50,16 +51,17 @@ use windmill_audit::ActionKind;
#[cfg(feature = "parquet")]
use windmill_common::s3_helpers::build_object_store_client;
use windmill_common::{
apps::ListAppQuery,
apps::{AppScriptId, ListAppQuery},
cache::{self, future::FutureCachedExt},
db::UserDB,
error::{to_anyhow, Error, JsonResult, Result},
jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode},
users::username_to_permissioned_as,
utils::{
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath,
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath
},
variables::{build_crypt, build_crypt_with_key_suffix},
worker::to_raw_value,
worker::{to_raw_value, CLOUD_HOSTED},
HUB_BASE_URL,
};
@@ -71,6 +73,7 @@ pub fn workspaced_service() -> Router {
.route("/list", get(list_apps))
.route("/list_search", get(list_search_apps))
.route("/get/p/*path", get(get_app))
.route("/get/lite/*path", get(get_app_lite))
.route("/get/draft/*path", get(get_app_w_draft))
.route("/secret_of/*path", get(get_secret_id))
.route("/get/v/*id", get(get_app_by_id))
@@ -81,6 +84,7 @@ pub fn workspaced_service() -> Router {
.route("/history/p/*path", get(get_app_history))
.route("/get_latest_version/*path", get(get_latest_version))
.route("/history_update/a/:id/v/:version", post(update_app_history))
.route("/custom_path_exists/*custom_path", get(custom_path_exists))
}
pub fn unauthed_service() -> Router {
@@ -90,13 +94,17 @@ pub fn unauthed_service() -> Router {
.route("/public_app/:secret", get(get_public_app_by_secret))
.route("/public_resource/*path", get(get_public_resource))
}
pub fn global_service() -> Router {
Router::new()
.route("/hub/list", get(list_hub_apps))
.route("/hub/get/:id", get(get_hub_app_by_id))
}
#[cfg(not(feature = "enterprise"))]
pub fn global_unauthed_service() -> Router {
Router::new()
}
#[derive(FromRow, Deserialize, Serialize)]
pub struct ListableApp {
pub id: i64,
@@ -147,21 +155,26 @@ pub struct AppWithLastVersionAndStarred {
pub starred: Option<bool>,
}
#[cfg(feature = "enterprise")]
#[derive(Serialize, FromRow)]
pub struct AppWithLastVersionAndWorkspace {
#[sqlx(flatten)]
#[serde(flatten)]
pub app: AppWithLastVersion,
pub workspace_id: String,
}
#[derive(Serialize, Deserialize, FromRow)]
pub struct AppWithLastVersionAndDraft {
pub id: i64,
pub path: String,
pub summary: String,
pub policy: sqlx::types::Json<Box<RawValue>>,
pub versions: Vec<i64>,
pub value: sqlx::types::Json<Box<RawValue>>,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub extra_perms: serde_json::Value,
#[sqlx(flatten)]
#[serde(flatten)]
pub app: AppWithLastVersion,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft: Option<sqlx::types::Json<Box<RawValue>>>,
#[serde(skip_serializing_if = "Option::is_none")]
pub draft_only: Option<bool>,
#[serde(skip_serializing_if = "Option::is_none")]
pub custom_path: Option<String>,
}
#[derive(Serialize)]
@@ -181,15 +194,16 @@ pub type StaticFields = HashMap<String, Box<RawValue>>;
pub type OneOfFields = HashMap<String, Vec<Box<RawValue>>>;
pub type AllowUserResources = Vec<String>;
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
#[serde(rename_all = "lowercase")]
pub enum ExecutionMode {
#[default]
Anonymous,
Publisher,
Viewer,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct PolicyTriggerableInputs {
static_inputs: StaticFields,
one_of_inputs: OneOfFields,
@@ -205,7 +219,7 @@ pub struct S3Input {
file_key_regex: String,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct Policy {
pub on_behalf_of: Option<String>,
pub on_behalf_of_email: Option<String>,
@@ -229,6 +243,7 @@ pub struct CreateApp {
pub policy: Policy,
pub draft_only: Option<bool>,
pub deployment_message: Option<String>,
pub custom_path: Option<String>,
}
#[derive(Deserialize)]
@@ -238,6 +253,7 @@ pub struct EditApp {
pub value: Option<sqlx::types::Json<Box<RawValue>>>,
pub policy: Option<Policy>,
pub deployment_message: Option<String>,
pub custom_path: Option<String>,
}
#[derive(Serialize, FromRow)]
@@ -398,6 +414,33 @@ async fn get_app(
Ok(Json(app))
}
async fn get_app_lite(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<AppWithLastVersion> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, coalesce(app_version_lite.value::json, app_version.value) as value,
app_version.created_at, app_version.created_by, NULL as starred
FROM app, app_version
LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
)
.bind(path.to_owned())
.bind(&w_id)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let app = not_found_if_none(app_o, "App", path)?;
Ok(Json(app))
}
async fn get_app_w_draft(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -408,7 +451,7 @@ async fn get_app_w_draft(
let app_o = sqlx::query_as::<_, AppWithLastVersionAndDraft>(
r#"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, app_version.value,
app.extra_perms, app_version.value, app.custom_path,
app_version.created_at, app_version.created_by,
app.draft_only, draft.value as "draft"
from app
@@ -515,6 +558,22 @@ async fn update_app_history(
return Ok(());
}
async fn custom_path_exists(
Extension(db): Extension<DB>,
Path((w_id, custom_path)): Path<(String, String)>,
) -> JsonResult<bool> {
let exists =
sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))",
custom_path,
if *CLOUD_HOSTED { Some(&w_id) } else { None }
)
.fetch_one(&db)
.await?.unwrap_or(false);
Ok(Json(exists))
}
async fn get_app_by_id(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -555,8 +614,9 @@ async fn get_public_app_by_secret(
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
null as extra_perms, app_version.value,
null as extra_perms, coalesce(app_version_lite.value::json, app_version.value::json) as value,
app_version.created_at, app_version.created_by from app, app_version
LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]")
.bind(&id)
.bind(&w_id)
@@ -598,6 +658,7 @@ async fn get_public_app_by_secret(
Ok(Json(app))
}
async fn get_public_resource(
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
@@ -680,6 +741,26 @@ async fn create_app(
)));
}
if let Some(custom_path) = &app.custom_path {
require_admin(authed.is_admin, &authed.username)?;
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))",
custom_path,
if *CLOUD_HOSTED { Some(&w_id) } else { None }
)
.fetch_one(&mut *tx)
.await?.unwrap_or(false);
if exists {
return Err(Error::BadRequest(format!(
"App with custom path {} already exists",
custom_path
)));
}
}
sqlx::query!(
"DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'",
&app.path,
@@ -690,13 +771,14 @@ async fn create_app(
let id = sqlx::query_scalar!(
"INSERT INTO app
(workspace_id, path, summary, policy, versions, draft_only)
VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id",
(workspace_id, path, summary, policy, versions, draft_only, custom_path)
VALUES ($1, $2, $3, $4, '{}', $5, $6) RETURNING id",
w_id,
app.path,
app.summary,
json!(app.policy),
app.draft_only,
app.custom_path,
)
.fetch_one(&mut *tx)
.await?;
@@ -899,7 +981,11 @@ async fn update_app(
let mut tx = user_db.clone().begin(&authed).await?;
let npath = if ns.policy.is_some() || ns.path.is_some() || ns.summary.is_some() {
let npath = if ns.policy.is_some()
|| ns.path.is_some()
|| ns.summary.is_some()
|| ns.custom_path.is_some()
{
let mut sqlb = SqlBuilder::update_table("app");
sqlb.and_where_eq("path", "?".bind(&path));
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
@@ -932,6 +1018,29 @@ async fn update_app(
sqlb.set_str("summary", nsummary);
}
if let Some(ncustom_path) = &ns.custom_path {
require_admin(authed.is_admin, &authed.username)?;
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))",
ncustom_path,
if *CLOUD_HOSTED { Some(&w_id) } else { None },
path,
w_id
)
.fetch_one(&mut *tx)
.await?.unwrap_or(false);
if exists {
return Err(Error::BadRequest(format!(
"App with custom path {} already exists",
ncustom_path
)));
}
sqlb.set_str("custom_path", ncustom_path);
}
if let Some(mut npolicy) = ns.policy {
npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username));
npolicy.on_behalf_of_email = Some(authed.email.clone());
@@ -1067,6 +1176,10 @@ async fn update_app(
#[derive(Debug, Deserialize, Clone)]
pub struct ExecuteApp {
/// The app version to execute. Fallback to `path` if not provided.
pub version: Option<i64>,
/// The app script id (from the `app_script` table) to execute.
pub id: Option<i64>,
pub args: HashMap<String, Box<RawValue>>,
// - script: script/<path>
// - flow: flow/<path>
@@ -1129,6 +1242,22 @@ async fn get_on_behalf_details_from_policy_and_authed(
Ok((username, permissioned_as, email))
}
/// Convert the triggerables from the old format to the new format.
fn empty_triggerables(mut policy: Policy) -> Policy {
use std::mem::take;
if let Some(triggerables) = take(&mut policy.triggerables) {
let mut triggerables_v2 = take(&mut policy.triggerables_v2).unwrap_or_default();
for (k, static_inputs) in triggerables.into_iter() {
triggerables_v2.insert(
k,
PolicyTriggerableInputs { static_inputs, ..Default::default() },
);
}
policy.triggerables_v2 = Some(triggerables_v2);
}
policy
}
async fn execute_component(
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
@@ -1151,99 +1280,136 @@ async fn execute_component(
};
let path = path.to_path();
let (arc_policy, policy): (Arc<Policy>, Policy);
let policy_triggerables_default = Default::default();
let policy = match payload.clone() {
// Two cases here:
// 1. The component is executed from the editor (i.e. in "preview" mode), then:
// - The policy is set to default (in `Viewer` execution mode).
// - The policy triggerables are built by the frontend and retrieved from the request
// payload.
// - In case of inline script, the `RawCode` from the request is pushed as is to the
// job queue.
// 2. Otherwise (i.e. "run" mode):
// - The policy and triggerables are fetched from the database.
// - In case of inline script, if an entry exists in the `app_script` table, push
// an `AppScript` job payload, as in (.1) otherwise.
let (policy, policy_triggerables) = match payload {
// 1. "preview" mode.
ExecuteApp {
force_viewer_static_fields: Some(static_fields),
force_viewer_one_of_fields: Some(one_of_fields),
force_viewer_static_fields: Some(static_inputs),
force_viewer_one_of_fields: Some(one_of_inputs),
force_viewer_allow_user_resources: Some(allow_user_resources),
..
} => {
let mut hm = HashMap::new();
if let Some(path) = payload.path.clone() {
hm.insert(
format!("{}:{path}", payload.component),
PolicyTriggerableInputs {
static_inputs: static_fields,
one_of_inputs: one_of_fields,
allow_user_resources,
},
);
} else {
hm.insert(
format!(
"{}:{}",
payload.component,
digest(payload.raw_code.clone().unwrap().content.as_str())
),
PolicyTriggerableInputs {
static_inputs: static_fields,
one_of_inputs: one_of_fields,
allow_user_resources,
},
);
}
Policy {
} => (
&Policy {
execution_mode: ExecutionMode::Viewer,
triggerables: None,
triggerables_v2: Some(hm),
on_behalf_of: None,
on_behalf_of_email: None,
s3_inputs: None,
}
}
..Default::default()
},
&PolicyTriggerableInputs {
static_inputs,
one_of_inputs,
allow_user_resources,
},
),
// 2. "run" mode.
_ => {
let policy_o = sqlx::query_scalar!(
"SELECT policy from app WHERE path = $1 AND workspace_id = $2",
// Policy is fetched from the database on app `path` and `workspace_id`.
let policy_fut = sqlx::query_scalar!(
"SELECT policy as \"policy: sqlx::types::Json<Box<RawValue>>\"
FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1",
path,
&w_id
&w_id,
)
.fetch_optional(&db)
.await?;
.map_err(Into::<Error>::into)
.map(|policy_o| Result::Ok(not_found_if_none(policy_o?, "App", path)?))
.map(|policy| Result::Ok(serde_json::from_str(policy?.get())?))
.map_ok(empty_triggerables);
let policy = not_found_if_none(policy_o, "App", path)?;
// 1. The app `version` is provided: cache the fetched policy.
// 2. Otherwise, always fetch the policy from the database.
let policy = if let Some(id) = payload.version {
let cache = cache::anon!({ u64 => Arc<Policy> } in "policy" <= 1000);
arc_policy = policy_fut
.map_ok(Arc::new)
.cached(cache, &(id as u64))
.await?;
&*arc_policy
} else {
policy = policy_fut.await?;
&policy
};
serde_json::from_value::<Policy>(policy).map_err(to_anyhow)?
// Compute the path for the triggerables map:
// - flow: `flow/<payload.path>`
// - script: `script/<payload.path>`
// - inline script: `rawscript/<sha256(raw_code.content)>`
let path = match &payload {
// flow or script: just use the `payload.path`.
ExecuteApp { path: Some(path), .. } => path,
// inline script: without entry in the `app_script` table.
ExecuteApp { raw_code: Some(raw_code), id: None, .. } => &digest(&raw_code.content),
// inline script: with an entry in the `app_script` table.
ExecuteApp { raw_code: Some(_), id: Some(id), .. } => {
let cache = cache::anon!({ u64 => Arc<String> } in "appscriptpath" <= 10000);
// `id` is unique, cache the result.
&*sqlx::query_scalar!(
"SELECT format('rawscript/%s', code_sha256) as \"path!: String\"
FROM app_script WHERE id = $1 LIMIT 1",
id
)
.fetch_one(&db)
.map_err(Into::<Error>::into)
.map_ok(Arc::new)
.cached(cache, &(*id as u64))
.await?
}
_ => unreachable!(),
};
// Retrieve the triggerables from the policy on `path` or `<component>:<path>`.
let triggerables_v2 = policy
.triggerables_v2
.as_ref()
.ok_or_else(|| Error::BadRequest(format!("Policy is missing triggerables")))?;
let policy_triggerables = triggerables_v2
.get(path) // start with `path` in case we can avoid the next` format!`.
.or_else(|| triggerables_v2.get(&format!("{}:{}", payload.component, &path)))
.or(match policy.execution_mode {
ExecutionMode::Viewer => Some(&policy_triggerables_default),
_ => None,
})
.ok_or_else(|| Error::BadRequest(format!("Path {path} forbidden by policy")))?;
(policy, policy_triggerables)
}
};
let (username, permissioned_as, email) =
get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?;
let (job_payload, (args, job_id), tag) = match payload {
ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => {
let content = &raw_code.content;
let payload = JobPayload::Code(raw_code.clone());
let path = digest(content);
let args = build_args(
policy,
&component,
path,
args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
(payload, args, None)
}
ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => {
let (payload, tag) = get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?;
let args = build_args(
policy,
&component,
path.to_string(),
args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
(payload, args, tag)
}
let (args, job_id) = build_args(
policy,
policy_triggerables,
payload.args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
let (job_payload, tag) = match (payload.path, payload.raw_code, payload.id) {
// flow or script:
(Some(path), None, None) => get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?,
// inline script: in "preview" mode or without entry in the `app_script` table.
(None, Some(raw_code), None) => (JobPayload::Code(raw_code), None),
// inline script: in "run" mode and with an entry in the `app_script` table.
(None, Some(RawCode { language, path, cache_ttl, .. }), Some(id)) => (
JobPayload::AppScript { id: AppScriptId(id), cache_ttl, language, path },
None,
),
_ => unreachable!(),
};
let tx = windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone());
@@ -1578,9 +1744,12 @@ async fn exists_app(
}
async fn build_args(
policy: Policy,
component: &str,
path: String,
policy: &Policy,
PolicyTriggerableInputs {
static_inputs,
one_of_inputs,
allow_user_resources,
}: &PolicyTriggerableInputs,
mut args: HashMap<String, Box<RawValue>>,
authed: Option<&ApiAuthed>,
user_db: &UserDB,
@@ -1588,54 +1757,6 @@ async fn build_args(
w_id: &str,
) -> Result<(PushArgsOwned, Option<Uuid>)> {
let mut job_id: Option<Uuid> = None;
let key = format!("{}:{}", component, &path);
let (static_inputs, one_of_inputs, allow_user_resources) = match policy {
Policy { triggerables_v2: Some(t), .. } => {
let PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources } = t
.get(&key)
.or_else(|| t.get(&path))
.map(|x| x.clone())
.or_else(|| {
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
Some(PolicyTriggerableInputs {
static_inputs: HashMap::new(),
one_of_inputs: HashMap::new(),
allow_user_resources: Vec::new(),
})
} else {
None
}
})
.ok_or_else(|| {
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
})?;
(static_inputs, one_of_inputs, allow_user_resources)
}
Policy { triggerables: Some(t), .. } => {
let static_inputs = t
.get(&key)
.or_else(|| t.get(&path))
.map(|x| x.clone())
.or_else(|| {
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
Some(HashMap::new())
} else {
None
}
})
.ok_or_else(|| {
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
})?;
(static_inputs, HashMap::new(), Vec::new())
}
_ => Err(Error::BadRequest(format!(
"Policy is missing triggerables for {}",
key
)))?,
};
let mut safe_args = HashMap::<String, Box<RawValue>>::new();
// tracing::error!("{:?}", allow_user_resources);
@@ -1684,16 +1805,16 @@ async fn build_args(
}
for (k, v) in one_of_inputs {
if safe_args.contains_key(&k) {
if safe_args.contains_key(k) {
continue;
}
if let Some(arg_val) = args.get(&k) {
if let Some(arg_val) = args.get(k) {
let arg_str = arg_val.get();
let options_str_vec = v.iter().map(|x| x.get()).collect::<Vec<&str>>();
if options_str_vec.contains(&arg_str) {
safe_args.insert(k.to_string(), arg_val.clone());
args.remove(&k);
args.remove(k);
continue;
}
@@ -1704,7 +1825,7 @@ async fn build_args(
.all(|x| options_str_vec.contains(&x.get()))
{
safe_args.insert(k.to_string(), arg_val.clone());
args.remove(&k);
args.remove(k);
continue;
}
}
backend/windmill-api/src/apps_ee.rs
+5
View File
@@ -0,0 +1,5 @@
use axum::Router;
pub fn global_unauthed_service() -> Router {
Router::new()
}
backend/windmill-api/src/lib.rs
+13
View File
@@ -64,6 +64,8 @@ mod indexer_ee;
mod inputs;
mod integration;
#[cfg(feature = "enterprise")]
mod apps_ee;
#[cfg(feature = "parquet")]
mod job_helpers_ee;
pub mod job_metrics;
@@ -343,6 +345,17 @@ pub async fn run_server(
)
.nest("/concurrency_groups", concurrency_groups::global_service())
.nest("/scripts_u", scripts::global_unauthed_service())
.nest("/apps_u", {
#[cfg(feature = "enterprise")]
{
apps_ee::global_unauthed_service()
}
#[cfg(not(feature = "enterprise"))]
{
Router::new()
}
})
.nest(
"/w/:workspace_id/apps_u",
apps::unauthed_service()
backend/windmill-api/src/settings.rs
+1 -1
View File
@@ -448,7 +448,7 @@ pub async fn get_critical_alerts(
Extension(db): Extension<DB>,
authed: ApiAuthed,
Query(params): Query<crate::utils::AlertQueryParams>,
) -> JsonResult<Vec<crate::utils::CriticalAlert>> {
) -> JsonResult<serde_json::Value> {
require_devops_role(&db, &authed.email).await?;
crate::utils::get_critical_alerts(db, params, None).await
backend/windmill-api/src/tracing_init.rs
+8 -6
View File
@@ -29,11 +29,13 @@ impl<B> OnResponse<B> for MyOnResponse {
_span: &tracing::Span,
) {
if *LOG_REQUESTS {
tracing::info!(
latency = latency.as_millis(),
status = response.status().as_u16(),
"response"
)
let latency = latency.as_millis();
let status = response.status().as_u16();
if response.status().is_success() {
tracing::info!(latency = latency, status = status, "response")
} else {
tracing::error!(latency = latency, status = status, "response")
}
}
}
}
@@ -67,7 +69,7 @@ impl<B> MakeSpan<B> for MyMakeSpan {
uri = %request.uri(),
username = field::Empty,
workspace_id = field::Empty,
trace_id = tracing_id,
traceId = tracing_id,
email = field::Empty,
)
}
backend/windmill-api/src/utils.rs
+70 -8
View File
@@ -10,6 +10,7 @@ use axum::{body::Body, response::Response};
use regex::Regex;
use serde::Deserialize;
use sqlx::{Postgres, Transaction};
use windmill_common::worker::CLOUD_HOSTED;
use windmill_common::{
auth::{is_devops_email, is_super_admin_email},
error::{self, Error},
@@ -206,11 +207,55 @@ pub async fn get_critical_alerts(
db: DB,
params: AlertQueryParams,
workspace_id: Option<String>,
) -> JsonResult<Vec<CriticalAlert>> {
) -> JsonResult<serde_json::Value> {
// Returning total rows and total pages
let page = params.page.unwrap_or(1).max(1);
let page_size = params.page_size.unwrap_or(10).min(100) as i64;
let offset = ((page - 1) * page_size as i32) as i64;
// Count total rows
let total_rows = if let Some(workspace_id) = &workspace_id {
if params.acknowledged.is_none() {
sqlx::query_scalar!(
"SELECT COUNT(*)
FROM alerts
WHERE workspace_id = $1",
workspace_id
)
.fetch_one(&db)
.await?
} else {
sqlx::query_scalar!(
"SELECT COUNT(*)
FROM alerts
WHERE workspace_id = $1 AND COALESCE(acknowledged_workspace, false) = $2",
workspace_id,
params.acknowledged
)
.fetch_one(&db)
.await?
}
} else {
if params.acknowledged.is_none() {
sqlx::query_scalar!(
"SELECT COUNT(*)
FROM alerts"
)
.fetch_one(&db)
.await?
} else {
sqlx::query_scalar!(
"SELECT COUNT(*)
FROM alerts
WHERE COALESCE(acknowledged, false) = $1",
params.acknowledged
)
.fetch_one(&db)
.await?
}
};
// Fetch paginated rows
let alerts = if let Some(workspace_id) = workspace_id {
// `workspace_id` is provided => workspace admin
if params.acknowledged.is_none() {
@@ -278,7 +323,14 @@ pub async fn get_critical_alerts(
}
};
Ok(Json(alerts))
let total_rows = total_rows.unwrap_or(0);
let total_pages = ((total_rows as f64) / (page_size as f64)).ceil() as i64;
Ok(Json(serde_json::json!({
"alerts": alerts,
"total_rows": total_rows,
"total_pages": total_pages
})))
}
#[cfg(feature = "enterprise")]
@@ -292,12 +344,17 @@ pub async fn acknowledge_critical_alert(
SET
acknowledged = true,
acknowledged_workspace = CASE
WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true
ELSE acknowledged_workspace
WHEN $3 THEN
CASE
WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true
ELSE acknowledged_workspace
END
ELSE true
END
WHERE id = $1",
id,
workspace_id
workspace_id,
*CLOUD_HOSTED
)
.execute(&db)
.await?;
@@ -320,12 +377,17 @@ pub async fn acknowledge_all_critical_alerts(
SET
acknowledged = true,
acknowledged_workspace = CASE
WHEN $1::text IS NOT NULL THEN true
ELSE acknowledged_workspace
WHEN $2 THEN
CASE
WHEN $1::text IS NOT NULL THEN true
ELSE acknowledged_workspace
END
ELSE true
END
WHERE ($1::text IS NOT NULL AND workspace_id = $1)
OR ($1::text IS NULL)",
workspace_id
workspace_id,
*CLOUD_HOSTED
)
.execute(&db)
.await?;
backend/windmill-api/src/workspaces.rs
+1 -1
View File
@@ -3111,7 +3111,7 @@ pub async fn get_critical_alerts(
Path(w_id): Path<String>,
authed: ApiAuthed,
Query(params): Query<crate::utils::AlertQueryParams>,
) -> JsonResult<Vec<crate::utils::CriticalAlert>> {
) -> JsonResult<serde_json::Value> {
require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, &db).await?;
crate::utils::get_critical_alerts(db, params, Some(w_id)).await
backend/windmill-common/Cargo.toml
+11
View File
@@ -13,6 +13,8 @@ flamegraph = ["dep:tracing-flame"]
loki = ["dep:tracing-loki"]
benchmark = []
parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"]
otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk",
"dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"]
[lib]
name = "windmill_common"
@@ -62,6 +64,15 @@ windmill-macros.workspace = true
semver.workspace = true
croner = "2.0.6"
quick_cache.workspace = true
pin-project-lite.workspace = true
opentelemetry-semantic-conventions = { workspace = true, optional = true }
opentelemetry-otlp = { workspace = true, optional = true }
opentelemetry_sdk = { workspace = true, optional = true }
opentelemetry = { workspace = true, optional = true }
tracing-opentelemetry = { workspace = true, optional = true }
opentelemetry-appender-tracing = { workspace = true, optional = true }
tonic = { workspace = true, optional = true }
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemalloc-ctl = { optional = true, workspace = true }
backend/windmill-common/src/apps.rs
+12 -1
View File
@@ -6,7 +6,18 @@
* LICENSE-AGPL for a copy of the license.
*/
use serde::Deserialize;
use serde::{Deserialize, Serialize};
/// Id in the `app_script` table.
#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)]
#[serde(transparent)]
pub struct AppScriptId(pub i64);
impl Into<u64> for AppScriptId {
fn into(self) -> u64 {
self.0 as u64
}
}
#[derive(Deserialize)]
pub struct ListAppQuery {
backend/windmill-common/src/cache.rs
+327 -61
View File
@@ -1,25 +1,149 @@
use crate::error;
use std::future::Future;
use std::hash::Hash;
use std::path::{Path, PathBuf};
use quick_cache::sync::Cache;
use quick_cache::Equivalent;
use serde::{Deserialize, Serialize};
use sqlx::PgExecutor;
pub use const_format::concatcp;
pub use lazy_static::lazy_static;
pub use quick_cache::sync::Cache;
/// Cache directory for windmill server/worker(s).
pub const CACHE_DIR: &str = "/tmp/windmill/cache/";
/// A file-system backed concurrent cache.
pub struct FsBackedCache<Key, Val> {
cache: Cache<Key, Val>,
root: &'static str,
}
impl<Key: Eq + Hash, Val: Clone + fs::Bundle> FsBackedCache<Key, Val> {
/// Create a new file-system backed cache with `items_capacity` capacity.
/// The cache will be stored in the `root` directory.
pub fn new(root: &'static str, items_capacity: usize) -> Self {
Self { cache: Cache::new(items_capacity), root }
}
/// Gets or inserts an item in the cache with key `key`.
pub async fn get_or_insert_async<'a, Q, F>(&'a self, key: &Q, with: F) -> error::Result<Val>
where
Q: Hash + Equivalent<Key> + ToOwned<Owned = Key> + Copy + Into<u64>,
F: Future<Output = error::Result<Val>>,
{
self.cache
.get_or_insert_async(
key,
fs::import_or_insert_with(self.root, (*key).into(), with),
)
.await
}
}
/// Like [`lazy_static`]`, but for file-system backed caches.
///
/// # Example
/// ```rust
/// use windmill_common::make_static;
///
/// make_static! {
/// /// String cache with a maximum capacity of 1000 items stored in the
/// /// "subdirectory" directory.
/// static ref CACHE: { u64 => String } in "subdirectory" <= 1000;
/// /// Another cache.
/// static ref ANOTHER_CACHE: { u64 => Vec<String> } in "another" <= 100;
/// }
/// ```
#[macro_export]
macro_rules! make_static {
{ $( $(#[$attr:meta])* static ref $name:ident: { $Key:ty => $Val:ty } in $root:literal <= $cap:literal; )+ } => {
$crate::cache::lazy_static! {
$(
$(#[$attr])*
static ref $name: $crate::cache::FsBackedCache<$Key, $Val> =
$crate::cache::FsBackedCache::new(
$crate::cache::concatcp!($crate::cache::CACHE_DIR, $root),
$cap
);
)+
}
};
}
// re-export:
pub use make_static;
/// Create an anonymous file-system backed cache for one-time use.
///
/// # Example
/// ```rust
/// use windmill_common::anon;
/// let cache = anon!({ u64 => String } in "subdirectory" <= 1000);
/// ```
#[macro_export]
macro_rules! anon {
({ $Key:ty => $Val:ty } in $root:literal <= $cap:literal) => {{
$crate::cache::make_static! {
static ref __ANON__: { $Key => $Val } in $root <= $cap;
}
&__ANON__
}};
}
// re-export:
pub use anon;
pub mod future {
use super::*;
/// Extension trait for futures that can be cached.
pub trait FutureCachedExt<Val: Clone + fs::Bundle>:
Future<Output = error::Result<Val>> + Sized
{
/// Get or insert the future result in the cache.
///
/// # Example
/// ```rust
/// use windmill_common::cache::{self, future::FutureCachedExt};
///
/// async {
/// let result = std::future::ready(Ok(42))
/// .cached(cache::anon!({ u64 => u64 } in "test" <= 1), &42)
/// .await;
///
/// assert_eq!(result.unwrap(), 42);
/// };
/// ```
fn cached<Key: Eq + Hash, Q>(
self,
cache: &FsBackedCache<Key, Val>,
key: &Q,
) -> impl Future<Output = error::Result<Val>>
where
Q: Hash + Equivalent<Key> + ToOwned<Owned = Key> + Copy + Into<u64>,
{
cache.get_or_insert_async(key, self)
}
}
impl<Val: Clone + fs::Bundle, F: Future<Output = error::Result<Val>> + Sized>
FutureCachedExt<Val> for F
{
}
}
pub mod flow {
use super::*;
use crate::flows::{FlowNodeId, FlowValue};
/// Cache directory for windmill server/worker(s) flow nodes.
pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "flow");
lazy_static::lazy_static! {
make_static! {
/// Flow node cache.
/// FIXME: This should be a static but [`Cache`] does not have a const constructor.
/// FIXME: Use `Arc<Val>` for cheap cloning.
static ref CACHE: Cache<FlowNodeId, Val> = Cache::new(1000);
static ref CACHE: { FlowNodeId => Val } in "flow" <= 1000;
}
/// Flow node cache value.
@@ -34,24 +158,36 @@ pub mod flow {
/// If not present, import from the file-system cache or fetch it from the database and write
/// it to the file system and cache.
/// This should be preferred over fetching the database directly.
pub async fn fetch_script(e: impl PgExecutor<'_>, node: FlowNodeId)
-> error::Result<(Option<String>, String)>
{
fetch(e, node).await.and_then(|Val { lock, code, .. }| Ok((lock, code.ok_or_else(|| {
error::Error::InternalErr(format!("Flow node ({:x}) isn't a script node.", node.0))
})?)))
pub async fn fetch_script(
e: impl PgExecutor<'_>,
node: FlowNodeId,
) -> error::Result<(Option<String>, String)> {
fetch(e, node).await.and_then(|Val { lock, code, .. }| {
Ok((
lock,
code.ok_or_else(|| {
error::Error::InternalErr(format!(
"Flow node ({:x}) isn't a script node.",
node.0
))
})?,
))
})
}
/// Fetch the flow node flow value referenced by `node` from the cache.
/// If not present, import from the file-system cache or fetch it from the database and write
/// it to the file system and cache.
/// This should be preferred over fetching the database directly.
pub async fn fetch_flow(e: impl PgExecutor<'_>, node: FlowNodeId)
-> error::Result<FlowValue>
{
fetch(e, node).await.and_then(|Val { flow, .. }| flow.ok_or_else(|| {
error::Error::InternalErr(format!("Flow node ({:x}) isn't a flow value node.", node.0))
}))
pub async fn fetch_flow(e: impl PgExecutor<'_>, node: FlowNodeId) -> error::Result<FlowValue> {
fetch(e, node).await.and_then(|Val { flow, .. }| {
flow.ok_or_else(|| {
error::Error::InternalErr(format!(
"Flow node ({:x}) isn't a flow value node.",
node.0
))
})
})
}
/// Fetch the flow node referenced by `node` from the cache.
@@ -62,9 +198,8 @@ pub mod flow {
// If not present, `get_or_insert_async` will lock the key until the future completes,
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
CACHE.get_or_insert_async(
&node,
fs::import_or_insert_with(CACHE_DIR, node.0 as u64, async {
CACHE
.get_or_insert_async(&node, async {
sqlx::query!(
"SELECT \
lock AS \"lock: String\", \
@@ -76,18 +211,24 @@ pub mod flow {
.fetch_one(e)
.await
.map_err(Into::into)
.and_then(|r| Ok(Val {
lock: r.lock.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
flow: match r.flow {
None => None,
Some(flow) => serde_json::from_str(&flow).map_err(|err| {
error::Error::InternalErr(format!("Unable to parse flow value: {err:?}"))
})?,
}
}))
.and_then(|r| {
Ok(Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
flow: match r.flow {
None => None,
Some(flow) => serde_json::from_str(&flow).map_err(|err| {
error::Error::InternalErr(format!(
"Unable to parse flow value: {err:?}"
))
})?,
},
})
})
})
).await
.await
}
// ----------------------------------------------------------------------------------------------
@@ -130,7 +271,11 @@ pub mod flow {
match item {
Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())),
Item::Code => Ok(self.code.as_ref().map(|s| s.as_bytes().to_vec())),
Item::Flow => Ok(self.flow.as_ref().map(|f| serde_json::to_vec(f)).transpose()?),
Item::Flow => Ok(self
.flow
.as_ref()
.map(|f| serde_json::to_vec(f))
.transpose()?),
}
}
}
@@ -140,14 +285,10 @@ pub mod script {
use super::*;
use crate::scripts::{ScriptHash, ScriptLang};
/// Cache directory for windmill server/worker(s) scripts.
pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "script");
lazy_static::lazy_static! {
make_static! {
/// Scripts cache.
/// FIXME: This should be a static but [`Cache`] does not have a const constructor.
/// FIXME: Use `Arc<Val>` for cheap cloning.
static ref CACHE: Cache<ScriptHash, Val> = Cache::new(1000);
static ref CACHE: { ScriptHash => Val } in "script" <= 1000;
}
/// Script cache value.
@@ -164,15 +305,16 @@ pub mod script {
/// If not present, import from the file-system cache or fetch it from the database and write
/// it to the file system and cache.
/// This should be preferred over fetching the database directly.
pub async fn fetch(e: impl PgExecutor<'_>, hash: ScriptHash, workspace_id: &str)
-> error::Result<Val>
{
pub async fn fetch(
e: impl PgExecutor<'_>,
hash: ScriptHash,
workspace_id: &str,
) -> error::Result<Val> {
// If not present, `get_or_insert_async` will lock the key until the future completes,
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
CACHE.get_or_insert_async(
&hash,
fs::import_or_insert_with(CACHE_DIR, hash.0 as u64, async {
CACHE
.get_or_insert_async(&hash, async {
sqlx::query!(
"SELECT \
lock AS \"lock: String\", \
@@ -188,15 +330,16 @@ pub mod script {
.await
.map_err(Into::into)
.map(|r| Val {
lock: r.lock.and_then(|x| if x.is_empty() { None } else { Some(x) }),
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
language: r.language,
envs: r.envs,
codebase: r.codebase,
})
})
)
.await
.await
}
// ----------------------------------------------------------------------------------------------
@@ -230,7 +373,104 @@ pub mod script {
match item {
Item::Lock => self.lock = Some(String::from_utf8(data)?),
Item::Code => self.code = String::from_utf8(data)?,
Item::Info => (self.language, self.envs, self.codebase) = serde_json::from_slice(&data)?,
Item::Info => {
(self.language, self.envs, self.codebase) = serde_json::from_slice(&data)?
}
}
Ok(())
}
fn export(&self, item: Self::Item) -> error::Result<Option<Vec<u8>>> {
match item {
Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())),
Item::Code => Ok(Some(self.code.as_bytes().to_vec())),
Item::Info => Ok(Some(serde_json::to_vec(&(
&self.language,
&self.envs,
&self.codebase,
))?)),
}
}
}
}
pub mod app {
use super::*;
use crate::apps::AppScriptId;
make_static! {
/// App scripts cache.
/// FIXME: Use `Arc<Val>` for cheap cloning.
static ref CACHE: { AppScriptId => Val } in "app" <= 1000;
}
/// App app script cache value.
#[derive(Debug, Clone, Default)]
pub struct Val {
pub lock: Option<String>,
pub code: String,
}
/// Fetch the app script referenced by `id` from the cache.
/// If not present, import from the file-system cache or fetch it from the database and write
/// it to the file system and cache.
/// This should be preferred over fetching the database directly.
pub async fn fetch_script(
e: impl PgExecutor<'_>,
id: AppScriptId,
) -> error::Result<(Option<String>, String)> {
// If not present, `get_or_insert_async` will lock the key until the future completes,
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
CACHE
.get_or_insert_async(&id, async {
sqlx::query!(
"SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1",
id.0,
)
.fetch_one(e)
.await
.map_err(Into::into)
.map(|r| Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
})
})
.await
.map(|Val { lock, code }| (lock, code))
}
// ----------------------------------------------------------------------------------------------
// impl `fs::Bundle` for `Val`.
#[derive(Copy, Clone)]
pub enum Item {
Lock,
Code,
}
impl fs::Item for Item {
fn path(&self, root: &Path) -> PathBuf {
match self {
Item::Lock => root.join("lock.txt"),
Item::Code => root.join("code.txt"),
}
}
}
impl fs::Bundle for Val {
type Item = Item;
fn items() -> &'static [Self::Item] {
&[Item::Lock, Item::Code]
}
fn import(&mut self, item: Self::Item, data: Vec<u8>) -> error::Result<()> {
match item {
Item::Lock => self.lock = Some(String::from_utf8(data)?),
Item::Code => self.code = String::from_utf8(data)?,
}
Ok(())
}
@@ -239,7 +479,6 @@ pub mod script {
match item {
Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())),
Item::Code => Ok(Some(self.code.as_bytes().to_vec())),
Item::Info => Ok(Some(serde_json::to_vec(&(&self.language, &self.envs, &self.codebase))?)),
}
}
}
@@ -248,8 +487,6 @@ pub mod script {
mod fs {
use super::*;
use std::future::Future;
use std::fs::{self, OpenOptions};
use std::io::{Read, Write};
@@ -272,8 +509,7 @@ mod fs {
}
/// Import or insert a bundle within the given combination of `{root}/{key}/`.
pub async fn import_or_insert_with<T, F>(root: &str, key: u64, f: F)
-> error::Result<T>
pub async fn import_or_insert_with<T, F>(root: &str, key: u64, f: F) -> error::Result<T>
where
T: Bundle,
F: Future<Output = error::Result<T>>,
@@ -287,8 +523,9 @@ mod fs {
let mut data = T::default();
for item in T::items() {
let mut buf = vec![];
let Ok(mut file) = OpenOptions::new().read(true).open(item.path(&path))
else { continue };
let Ok(mut file) = OpenOptions::new().read(true).open(item.path(&path)) else {
continue;
};
file.read_to_end(&mut buf)?;
data.import(*item, buf)?;
}
@@ -299,7 +536,7 @@ mod fs {
Ok(data) => return Ok(data),
Err(err) => tracing::warn!(
"Failed to import from file-system, fetch source..: {path:?}: {err:?}"
)
),
}
}
// Cache path doesn't exist or import failed, generate the content.
@@ -308,9 +545,13 @@ mod fs {
fs::create_dir_all(&path)?;
// Write the generated data to the file.
for item in T::items() {
let Some(buf) = data.export(*item)?
else { continue };
let mut file = OpenOptions::new().write(true).create(true).open(item.path(&path))?;
let Some(buf) = data.export(*item)? else {
continue;
};
let mut file = OpenOptions::new()
.write(true)
.create(true)
.open(item.path(&path))?;
file.write_all(&buf)?;
}
tracing::debug!("Exported to file-system: {:?}", path);
@@ -324,4 +565,29 @@ mod fs {
}
Ok(data)
}
// Auto-implement `Bundle` for all `serde` serializable types.
impl Item for () {
fn path(&self, root: &Path) -> PathBuf {
root.join("self.json")
}
}
impl<T: for<'de> Deserialize<'de> + Serialize + Default> Bundle for T {
type Item = ();
fn items() -> &'static [Self::Item] {
&[()]
}
fn import(&mut self, _: Self::Item, data: Vec<u8>) -> error::Result<()> {
*self = serde_json::from_slice(&data)?;
Ok(())
}
fn export(&self, _: Self::Item) -> error::Result<Option<Vec<u8>>> {
Ok(Some(serde_json::to_vec(self)?))
}
}
}
backend/windmill-common/src/flow_status.rs
+2 -2
View File
@@ -17,7 +17,7 @@ use crate::flows::FlowValue;
const MINUTES: Duration = Duration::from_secs(60);
const HOURS: Duration = MINUTES.saturating_mul(60);
pub const MAX_RETRY_ATTEMPTS: u16 = 1000;
pub const MAX_RETRY_ATTEMPTS: u32 = u32::MAX;
pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6);
pub fn is_retry_default(v: &RetryStatus) -> bool {
@@ -48,7 +48,7 @@ pub struct FlowStatus {
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
#[serde(default)]
pub struct RetryStatus {
pub fail_count: u16,
pub fail_count: u32,
pub failed_jobs: Vec<Uuid>,
}
backend/windmill-common/src/flows.rs
+75 -23
View File
@@ -147,7 +147,7 @@ impl Retry {
/// Takes the number of previous retries and returns the interval until the next retry if any.
///
/// May return [`Duration::ZERO`] to retry immediately.
pub fn interval(&self, previous_attempts: u16, silent: bool) -> Option<Duration> {
pub fn interval(&self, previous_attempts: u32, silent: bool) -> Option<Duration> {
let Self { constant, exponential } = self;
if previous_attempts < constant.attempts {
@@ -178,7 +178,7 @@ impl Retry {
self.constant.attempts != 0 || self.exponential.attempts != 0
}
pub fn max_attempts(&self) -> u16 {
pub fn max_attempts(&self) -> u32 {
self.constant
.attempts
.saturating_add(self.exponential.attempts)
@@ -194,7 +194,7 @@ impl Retry {
#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)]
#[serde(default)]
pub struct ConstantDelay {
pub attempts: u16,
pub attempts: u32,
pub seconds: u16,
}
@@ -202,7 +202,7 @@ pub struct ConstantDelay {
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
#[serde(default)]
pub struct ExponentialDelay {
pub attempts: u16,
pub attempts: u32,
pub multiplier: u16,
pub seconds: u16,
pub random_factor: Option<i8>, // percentage, defaults to 0 for no jitter
@@ -407,6 +407,12 @@ pub enum InputTransform {
#[serde(transparent)]
pub struct FlowNodeId(pub i64);
impl Into<u64> for FlowNodeId {
fn into(self) -> u64 {
self.0 as u64
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Branch {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -713,10 +719,14 @@ pub async fn resolve_maybe_value<T>(
workspace_id: &str,
with_code: bool,
maybe: Option<T>,
value_mut: impl FnOnce(&mut T) -> Option<&mut Json<Box<JsonRawValue>>>
value_mut: impl FnOnce(&mut T) -> Option<&mut Json<Box<JsonRawValue>>>,
) -> Result<Option<T>, Error> {
let Some(mut container) = maybe else { return Ok(None); };
let Some(value) = value_mut(&mut container) else { return Ok(Some(container)); };
let Some(mut container) = maybe else {
return Ok(None);
};
let Some(value) = value_mut(&mut container) else {
return Ok(Some(container));
};
resolve_value(e, workspace_id, &mut value.0, with_code).await?;
Ok(Some(container))
}
@@ -728,8 +738,9 @@ pub async fn resolve_value(
value: &mut Box<JsonRawValue>,
with_code: bool,
) -> Result<(), Error> {
let mut val = serde_json::from_str::<FlowValue>(value.get())
.map_err(|err| Error::InternalErr(format!("resolve: Failed to parse flow value: {}", err)))?;
let mut val = serde_json::from_str::<FlowValue>(value.get()).map_err(|err| {
Error::InternalErr(format!("resolve: Failed to parse flow value: {}", err))
})?;
for module in &mut val.modules {
resolve_module(e, workspace_id, &mut module.value, with_code).await?;
}
@@ -746,16 +757,29 @@ pub async fn resolve_module(
) -> Result<(), Error> {
use FlowModuleValue::*;
let mut val = serde_json::from_str::<FlowModuleValue>(value.get())
.map_err(|err| Error::InternalErr(format!("resolve: Failed to parse flow module value: {}", err)))?;
let mut val = serde_json::from_str::<FlowModuleValue>(value.get()).map_err(|err| {
Error::InternalErr(format!(
"resolve: Failed to parse flow module value: {}",
err
))
})?;
match &mut val {
FlowScript { .. } => {
// In order to avoid an unnecessary `.clone()` of `val`, take ownership of it's content
// using `std::mem::replace`.
let FlowScript {
input_transforms, id, tag, language,
custom_concurrency_key, concurrent_limit, concurrency_time_window_s, is_trigger
} = std::mem::replace(&mut val, Identity) else { unreachable!() };
input_transforms,
id,
tag,
language,
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
is_trigger,
} = std::mem::replace(&mut val, Identity)
else {
unreachable!()
};
// Load script lock file and code content.
let (lock, content) = if !with_code {
(Some("...".to_string()), "...".to_string())
@@ -763,22 +787,44 @@ pub async fn resolve_module(
cache::flow::fetch_script(e, id).await?
};
val = RawScript {
input_transforms, content, lock, path: None, tag, language, custom_concurrency_key,
concurrent_limit, concurrency_time_window_s, is_trigger
input_transforms,
content,
lock,
path: None,
tag,
language,
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
is_trigger,
};
},
ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => {
}
ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => {
resolve_modules(e, workspace_id, modules, modules_node.take(), with_code).await?;
},
}
BranchOne { branches, default, default_node } => {
resolve_modules(e, workspace_id, default, default_node.take(), with_code).await?;
for branch in branches {
resolve_modules(e, workspace_id, &mut branch.modules, branch.modules_node.take(), with_code).await?;
resolve_modules(
e,
workspace_id,
&mut branch.modules,
branch.modules_node.take(),
with_code,
)
.await?;
}
},
}
BranchAll { branches, .. } => {
for branch in branches {
resolve_modules(e, workspace_id, &mut branch.modules, branch.modules_node.take(), with_code).await?;
resolve_modules(
e,
workspace_id,
&mut branch.modules,
branch.modules_node.take(),
with_code,
)
.await?;
}
}
_ => {}
@@ -801,7 +847,13 @@ pub async fn resolve_modules(
.map(|flow| flow.modules)?;
}
for module in modules.iter_mut() {
Box::pin(resolve_module(e, workspace_id, &mut module.value, with_code)).await?;
Box::pin(resolve_module(
e,
workspace_id,
&mut module.value,
with_code,
))
.await?;
}
Ok(())
}
backend/windmill-common/src/global_settings.rs
+5 -1
View File
@@ -35,8 +35,9 @@ pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui";
pub const DEV_INSTANCE_SETTING: &str = "dev_instance";
pub const JWT_SECRET_SETTING: &str = "jwt_secret";
pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";
pub const OTEL_SETTING: &str = "otel";
pub const ENV_SETTINGS: [&str; 51] = [
pub const ENV_SETTINGS: [&str; 54] = [
"DISABLE_NSJAIL",
"MODE",
"NUM_WORKERS",
@@ -88,4 +89,7 @@ pub const ENV_SETTINGS: [&str; 51] = [
"WORKER_GROUP",
"SAML_METADATA",
"INSTANCE_IS_DEV",
"OTEL_METRICS",
"OTEL_TRACING",
"OTEL_LOGS",
];
backend/windmill-common/src/job_metrics.rs
+2 -1
View File
@@ -1,4 +1,4 @@
use crate::{db::DB, error};
use crate::{db::DB, error, utils::WarnAfterExt};
use serde::{Deserialize, Serialize};
use uuid::Uuid;
@@ -90,6 +90,7 @@ pub async fn register_metric_for_job(
.bind(timeseries_int)
.bind(timeseries_float)
.execute(db)
.warn_after_seconds(1)
.await?;
Ok(metric_id)
backend/windmill-common/src/jobs.rs
+8
View File
@@ -14,6 +14,7 @@ pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE";
pub const PREPROCESSOR_FAKE_ENTRYPOINT: &str = "__WM_PREPROCESSOR";
use crate::{
apps::AppScriptId,
error::{self, to_anyhow, Error},
flow_status::{FlowStatus, RestartedFrom},
flows::{FlowNodeId, FlowValue, Retry},
@@ -41,6 +42,7 @@ pub enum JobKind {
DeploymentCallback,
FlowScript,
FlowNode,
AppScript,
}
#[derive(sqlx::FromRow, Debug, Serialize, Clone)]
@@ -278,6 +280,12 @@ pub enum JobPayload {
id: FlowNodeId, // flow_node(id).
path: String, // flow node inner path (e.g. `outer/branchall-42`).
},
AppScript {
id: AppScriptId, // app_script(id).
path: Option<String>,
language: ScriptLang,
cache_ttl: Option<i32>,
},
Code(RawCode),
Dependencies {
path: String,
backend/windmill-common/src/lib.rs
+8 -2
View File
@@ -36,20 +36,20 @@ pub mod job_s3_helpers_ee;
pub mod jobs;
pub mod more_serde;
pub mod oauth2;
pub mod otel_ee;
pub mod queue;
pub mod s3_helpers;
pub mod schedule;
pub mod scripts;
pub mod server;
pub mod stats_ee;
pub mod tracing_init;
pub mod users;
pub mod utils;
pub mod variables;
pub mod worker;
pub mod workspaces;
pub mod tracing_init;
pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5;
pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5;
@@ -87,6 +87,12 @@ lazy_static::lazy_static! {
.unwrap_or_else(|| SocketAddr::from(([0, 0, 0, 0], *METRICS_PORT)));
pub static ref METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("METRICS_PORT").is_ok() || std::env::var("METRICS_ADDR").is_ok());
pub static ref OTEL_METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_METRICS").is_ok());
pub static ref OTEL_TRACING_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_TRACING").is_ok());
pub static ref OTEL_LOGS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_LOGS").is_ok());
pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false);
pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false);
backend/windmill-common/src/otel_ee.rs
+58
View File
@@ -0,0 +1,58 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{jobs::QueuedJob, utils::Mode};
use uuid::Uuid;
pub fn set_span_parent(_span: &tracing::Span, _rj: &Uuid) {}
#[cfg(not(all(feature = "otel", feature = "enterprise")))]
pub(crate) type OtelProvider = Option<()>;
#[cfg(all(feature = "otel", feature = "enterprise"))]
pub(crate) type OtelProvider = Option<opentelemetry_sdk::metrics::SdkMeterProvider>;
#[cfg(not(feature = "otel"))]
pub fn otel_ctx() -> () {}
#[cfg(feature = "otel")]
#[inline(always)]
pub fn otel_ctx() -> opentelemetry::Context {
opentelemetry::Context::current()
}
#[cfg(not(feature = "otel"))]
impl<T: Sized> FutureExt for T {}
#[cfg(not(feature = "otel"))]
pub trait FutureExt: Sized {
fn with_context(self, _otel_cx: ()) -> Self {
self
}
}
use tracing_subscriber::EnvFilter;
pub(crate) fn init_logs_bridge(_mode: &Mode, _hostname: &str, _env: &str) -> Option<EnvFilter> {
None
}
#[cfg(all(feature = "otel", feature = "enterprise"))]
pub(crate) fn init_otlp_tracer(
_mode: &Mode,
_hostname: &str,
_env: &str,
) -> Option<opentelemetry_sdk::trace::Tracer> {
None
}
pub(crate) fn init_meter_provider(_mode: &Mode, _hostname: &str, _env: &str) -> OtelProvider {
None
}
pub fn add_root_flow_job_to_otlp(_queued_job: &QueuedJob, _success: bool) {}
backend/windmill-common/src/scripts.rs
+6
View File
@@ -75,6 +75,12 @@ impl ScriptLang {
#[sqlx(transparent)]
pub struct ScriptHash(pub i64);
impl Into<u64> for ScriptHash {
fn into(self) -> u64 {
self.0 as u64
}
}
#[derive(PartialEq, sqlx::Type)]
#[sqlx(transparent, no_pg_array)]
pub struct ScriptHashes(pub Vec<i64>);
backend/windmill-common/src/tracing_init.rs
+58 -37
View File
@@ -7,13 +7,23 @@
*/
use const_format::concatcp;
use std::{
collections::HashMap,
sync::{Arc, RwLock},
};
use tracing::Event;
use tracing_appender::non_blocking::{NonBlockingBuilder, WorkerGuard};
use tracing_subscriber::layer::Context;
use tracing_subscriber::{
filter::Targets,
fmt::{format, Layer},
prelude::*,
EnvFilter,
};
use crate::utils::Mode;
fn json_layer<S>() -> Layer<S, format::JsonFields, format::Format<format::Json>> {
tracing_subscriber::fmt::layer()
.json()
@@ -34,7 +44,11 @@ pub const LOGS_SERVICE: &str = "logs/services/";
pub const TMP_WINDMILL_LOGS_SERVICE: &str = concatcp!("/tmp/windmill/", LOGS_SERVICE);
pub fn initialize_tracing(hostname: &str) -> WorkerGuard {
pub fn initialize_tracing(
hostname: &str,
mode: &Mode,
environment: &str,
) -> (WorkerGuard, crate::otel_ee::OtelProvider) {
let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into());
if std::env::var("RUST_LOG").is_ok_and(|x| x == "debug" || x == "info") {
@@ -44,7 +58,17 @@ pub fn initialize_tracing(hostname: &str) -> WorkerGuard {
)
}
let env_filter = EnvFilter::from_default_env();
let meter_provider = crate::otel_ee::init_meter_provider(mode, hostname, environment);
#[cfg(all(feature = "otel", feature = "enterprise"))]
let opentelemetry = crate::otel_ee::init_otlp_tracer(mode, hostname, environment)
.map(|x| tracing_opentelemetry::layer().with_tracer(x));
#[cfg(not(all(feature = "otel", feature = "enterprise")))]
let opentelemetry: Option<EnvFilter> = None;
let logs_bridge = crate::otel_ee::init_logs_bridge(&mode, hostname, environment);
use tracing_appender::rolling::{RollingFileAppender, Rotation};
let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname);
@@ -61,39 +85,59 @@ pub fn initialize_tracing(hostname: &str) -> WorkerGuard {
.finish(file_appender);
let stdout_and_log_file_writer = std::io::stdout.and(log_file_writer);
let ts_base = tracing_subscriber::registry().with(env_filter);
// let job_logs_filter = tracing_subscriber::filter::Targets::new()
// .with_target("windmill:job_log", tracing::Level::TRACE);
#[cfg(feature = "loki")]
let ts_base = {
let (layer, task) = tracing_loki::builder()
.build_url(reqwest::Url::parse("http://127.0.0.1:3100").unwrap())
.expect("build loki url");
tokio::spawn(task);
ts_base.with(layer)
};
let env_filter = EnvFilter::builder()
.with_default_directive(tracing::level_filters::LevelFilter::ERROR.into())
.from_env_lossy();
let ts_base = tracing_subscriber::registry().with(env_filter);
match *JSON_FMT {
true => ts_base
.with(logs_bridge)
.with(opentelemetry)
// .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap()))
.with(
json_layer()
.with_writer(stdout_and_log_file_writer)
.flatten_event(true),
.flatten_event(true)
.with_filter(
Targets::new()
.with_target(
"windmill:job_log",
tracing::level_filters::LevelFilter::OFF,
)
.with_default(tracing::level_filters::LevelFilter::INFO),
),
)
.with(CountingLayer::new())
.init(),
false => ts_base
.with(logs_bridge)
.with(opentelemetry)
// .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap()))
.with(
compact_layer()
.with_writer(stdout_and_log_file_writer)
.with_ansi(style.to_lowercase() != "never")
.with_file(true)
.with_line_number(true)
.with_target(false),
.with_target(false)
.with_filter(
Targets::new()
.with_target(
"windmill:job_log",
tracing::level_filters::LevelFilter::OFF,
)
.with_default(tracing::level_filters::LevelFilter::INFO),
),
)
.with(CountingLayer::new())
.init(),
}
_guard
(_guard, meter_provider)
}
#[cfg(feature = "flamegraph")]
@@ -112,13 +156,6 @@ pub fn setup_flamegraph() -> impl Drop {
_guard
}
use std::{
collections::HashMap,
sync::{Arc, RwLock},
};
use tracing::Event;
use tracing_subscriber::layer::Context;
lazy_static::lazy_static! {
pub static ref LOG_COUNTING_BY_MIN: Arc<RwLock<HashMap<String, LogCounter>>> = Arc::new(RwLock::new(HashMap::new()));
}
@@ -144,22 +181,6 @@ impl CountingLayer {
}
}
// impl CountingLayer {
// pub fn new() -> Self {
// CountingLayer { counter: Arc::new(Mutex::new(LogCounter::new())) }
// }
// pub fn get_counts(&self) -> (usize, usize) {
// let counter = self.counter.lock().unwrap();
// (counter.non_error_count, counter.error_count)
// }
// pub fn reset_counts(&self) {
// let mut counter = self.counter.lock().unwrap();
// counter.reset();
// }
// }
pub const LOG_TIMESTAMP_FMT: &str = "%Y-%m-%d-%H-%M";
impl<S> tracing_subscriber::Layer<S> for CountingLayer
backend/windmill-common/src/utils.rs
+92 -9
View File
@@ -34,8 +34,8 @@ pub const GIT_VERSION: &str =
git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
use crate::CRITICAL_ALERT_MUTE_UI_ENABLED;
use std::panic::{self, AssertUnwindSafe, Location};
use std::sync::atomic::Ordering;
use std::panic::{self, AssertUnwindSafe};
use crate::worker::CLOUD_HOSTED;
@@ -78,14 +78,18 @@ pub fn require_admin(is_admin: bool, username: &str) -> Result<()> {
}
}
pub async fn require_admin_or_devops(is_admin: bool, username: &str, email: &str, db: &DB) -> Result<()> {
pub async fn require_admin_or_devops(
is_admin: bool,
username: &str,
email: &str,
db: &DB,
) -> Result<()> {
if !is_admin {
if !is_devops_email(db, email).await? {
return Err(Error::RequireAdmin(username.to_string()));
}
}
Ok(())
}
pub fn hostname() -> String {
@@ -94,7 +98,7 @@ pub fn hostname() -> String {
.to_str()
.map(|x| x.to_string())
.unwrap_or_else(|| rd_string(5))
})
})
}
pub fn paginate(pagination: Pagination) -> (usize, usize) {
@@ -440,9 +444,7 @@ impl ScheduleType {
Some("v2") | Some(_) => {
// Use Croner for v2
let schedule_type_result = panic::catch_unwind(AssertUnwindSafe(|| {
Cron::new(schedule_str)
.with_seconds_optional()
.parse()
Cron::new(schedule_str).with_seconds_optional().parse()
}))
.map_err(|_| {
tracing::error!(
@@ -478,8 +480,14 @@ impl ScheduleType {
}
if let Err(e) = result {
tracing::error!("An error occurred while finding the next occurrence: {:?}", e);
return Err(Error::BadRequest(format!("cron: error during find_next_occurrence: {:?}", e)));
tracing::error!(
"An error occurred while finding the next occurrence: {:?}",
e
);
return Err(Error::BadRequest(format!(
"cron: error during find_next_occurrence: {:?}",
e
)));
}
}
@@ -526,3 +534,78 @@ impl ScheduleType {
Ok(events)
}
}
use std::future::Future;
use std::pin::Pin;
use std::task::{Context as TContext, Poll};
use tokio::time::{self, Duration, Sleep};
use pin_project_lite::pin_project;
pub trait WarnAfterExt: Future + Sized {
/// Warns if the future takes longer than the specified number of seconds to complete.
#[track_caller]
fn warn_after_seconds(self, seconds: u8) -> WarnAfterFuture<Self> {
let caller = Location::caller();
let location = format!("{}:{}", caller.file(), caller.line());
WarnAfterFuture {
future: self,
timeout: time::sleep(Duration::from_secs(seconds as u64)),
warned: false,
start_time: std::time::Instant::now(),
location: location,
seconds,
}
}
}
// Blanket implementation for all futures.
impl<F: Future> WarnAfterExt for F {}
pin_project! {
/// A future that wraps another future and prints a warning if it takes too long.
pub struct WarnAfterFuture<F> {
#[pin]
future: F,
#[pin]
timeout: Sleep,
warned: bool,
location: String,
start_time: std::time::Instant,
seconds: u8,
}
}
impl<F: Future> Future for WarnAfterFuture<F> {
type Output = F::Output;
fn poll(self: Pin<&mut Self>, cx: &mut TContext<'_>) -> Poll<Self::Output> {
let this = self.project();
// Poll the timeout future to check if it has elapsed.
if !*this.warned {
if this.timeout.poll(cx).is_ready() {
tracing::warn!(location = this.location, "SLOW_QUERY: query to db taking longer than expected (> {} seconds). This is a sign the database is under heavy load, query is too heavy or database is undersized",
this.seconds,
);
*this.warned = true;
}
}
// Poll the wrapped future.
match this.future.poll(cx) {
Poll::Ready(output) => {
if *this.warned {
let elapsed = this.start_time.elapsed();
tracing::warn!(
location = this.location,
"SLOW_QUERY: completed with total duration: {:.2?}",
elapsed
);
}
Poll::Ready(output)
}
Poll::Pending => Poll::Pending,
}
}
}
backend/windmill-common/src/worker.rs
+12 -3
View File
@@ -17,7 +17,9 @@ use std::{
use tokio::sync::RwLock;
use windmill_macros::annotations;
use crate::{error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB};
use crate::{
error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB,
};
lazy_static::lazy_static! {
pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
@@ -352,6 +354,11 @@ pub struct SqlAnnotations {
pub return_last_result: bool,
}
#[annotations("#")]
pub struct BashAnnotations {
pub docker: bool,
}
pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
if tokio::fs::metadata(&bin_path).await.is_ok() {
(true, format!("loaded from local cache: {}\n", bin_path))
@@ -577,8 +584,10 @@ pub fn get_windmill_memory_usage() -> Option<i64> {
}
}
pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(executor: E) -> bool {
use crate::utils::{GIT_VERSION, GIT_SEM_VERSION};
pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(
executor: E,
) -> bool {
use crate::utils::{GIT_SEM_VERSION, GIT_VERSION};
// fetch all pings with a different version than self from the last 5 minutes.
let pings = sqlx::query_scalar!(
backend/windmill-queue/Cargo.toml
+2 -1
View File
@@ -43,4 +43,5 @@ bigdecimal.workspace = true
axum.workspace = true
serde_urlencoded.workspace = true
regex.workspace = true
backon.workspace = true
backon.workspace = true
opentelemetry.workspace = true
backend/windmill-queue/src/jobs.rs
+26 -10
View File
@@ -32,15 +32,14 @@ use sqlx::{types::Json, FromRow, Pool, Postgres, Transaction};
#[cfg(feature = "benchmark")]
use std::time::Instant;
use tokio::{sync::RwLock, time::sleep};
use tracing::{instrument, Instrument};
use ulid::Ulid;
use uuid::Uuid;
use windmill_audit::audit_ee::{audit_log, AuditAuthor};
use windmill_audit::ActionKind;
use windmill_common::{
cache,
auth::{fetch_authed_from_permissioned_as, permissioned_as_to_username},
cache,
db::{Authed, UserDB},
error::{self, to_anyhow, Error},
flow_status::{
@@ -57,7 +56,7 @@ use windmill_common::{
schedule::Schedule,
scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang},
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
utils::{not_found_if_none, report_critical_error, StripPath},
utils::{not_found_if_none, report_critical_error, StripPath, WarnAfterExt},
worker::{
to_raw_value, CLOUD_HOSTED, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES,
DISABLE_FLOW_SCRIPT, MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432, NO_LOGS,
@@ -318,6 +317,7 @@ pub async fn append_logs(
workspace.as_ref(),
)
.execute(db.borrow())
.warn_after_seconds(1)
.await
{
tracing::error!(%job_id, %err, "error updating logs for large_log job {job_id}: {err}");
@@ -453,7 +453,6 @@ where
}
}
#[instrument(level = "trace", skip_all)]
pub async fn add_completed_job_error(
db: &Pool<Postgres>,
queued_job: &QueuedJob,
@@ -509,7 +508,6 @@ lazy_static::lazy_static! {
pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option<String> = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok();
}
#[instrument(level = "trace", skip_all, name = "add_completed_job")]
pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
db: &Pool<Postgres>,
queued_job: &QueuedJob,
@@ -642,9 +640,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
.fetch_one(&mut *tx)
.await
.map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e:#}")))?;
// tracing::error!("2 {:?}", start.elapsed());
// add_time!(bench, "add_completed_job query END");
if !queued_job.is_flow_step {
if _duration > 500
@@ -1258,7 +1254,6 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>
Ok(())
}
#[instrument(level = "trace", skip_all)]
pub async fn handle_maybe_scheduled_job<'c>(
db: &Pool<Postgres>,
job: &QueuedJob,
@@ -2428,7 +2423,6 @@ async fn extract_result_from_job_result(
}
}
#[instrument(level = "trace", skip_all)]
pub async fn delete_job<'c>(
mut tx: Transaction<'c, Postgres>,
w_id: &str,
@@ -3228,6 +3222,26 @@ pub async fn push<'c, 'd>(
None,
)
},
JobPayload::AppScript {
id, // app_script(id).
path,
language,
cache_ttl,
} => (
Some(id.0),
path,
None,
JobKind::AppScript,
None,
None,
Some(language),
None,
None,
None,
cache_ttl,
None,
None,
),
JobPayload::ScriptHub { path } => {
if path == "hub/7771/slack" || path == "hub/7836/slack" {
permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string();
@@ -3851,6 +3865,7 @@ pub async fn push<'c, 'd>(
tag,
)
.execute(&mut *tx)
.warn_after_seconds(1)
.await?;
let (raw_code, raw_lock, raw_flow) = if !*MIN_VERSION_IS_AT_LEAST_1_427.read().await {
@@ -3901,6 +3916,7 @@ pub async fn push<'c, 'd>(
final_priority,
)
.fetch_one(&mut *tx)
.warn_after_seconds(1)
.await
.map_err(|e| Error::InternalErr(format!("Could not insert into queue {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}: {e:#}")))?;
@@ -3986,6 +4002,7 @@ pub async fn push<'c, 'd>(
JobKind::DeploymentCallback => "jobs.run.deployment_callback",
JobKind::FlowScript => "jobs.run.flow_script",
JobKind::FlowNode => "jobs.run.flow_node",
JobKind::AppScript => "jobs.run.app_script",
};
let audit_author = if format!("u/{user}") != permissioned_as && user != permissioned_as {
@@ -4015,7 +4032,6 @@ pub async fn push<'c, 'd>(
script_path.as_ref().map(|x| x.as_str()),
Some(hm),
)
.instrument(tracing::info_span!("job_run", email = &email))
.await?;
}
backend/windmill-worker/Cargo.toml
+11 -1
View File
@@ -18,7 +18,10 @@ parquet = ["windmill-common/parquet", "dep:object_store"]
flow_testing = []
cloud = []
sqlx = []
deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", "dep:deno_ast", "dep:deno_tls"]
deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core",
"dep:deno_ast", "dep:deno_tls", "dep:deno_permissions"]
otel = ["windmill-common/otel", "dep:opentelemetry"]
dind = ["dep:bollard"]
[dependencies]
windmill-queue.workspace = true
@@ -68,6 +71,8 @@ deno_url = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
deno_ast = { workspace = true, optional = true }
deno_tls = { workspace = true, optional = true }
deno_permissions = { workspace = true, optional = true }
postgres-native-tls.workspace = true
native-tls.workspace = true
mysql_async.workspace = true
@@ -92,6 +97,9 @@ yaml-rust.workspace = true
swc_ecma_parser.workspace = true
backon.workspace = true
opentelemetry = { workspace = true, optional = true }
bollard = { workspace = true, optional = true }
[build-dependencies]
deno_fetch = { workspace = true, optional = true }
deno_webidl = { workspace = true, optional = true }
@@ -102,4 +110,6 @@ deno_url = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
deno_ast = { workspace = true, optional = true }
deno_tls = { workspace = true, optional = true }
deno_permissions = { workspace = true, optional = true }
zstd.workspace = true
backend/windmill-worker/build.rs
+8 -7
View File
@@ -25,7 +25,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
_url: &deno_core::url::Url,
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -34,7 +34,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
_p: &'a std::path::Path,
_api_name: &str,
) -> Result<Cow<'a, Path>, deno_core::error::AnyError> {
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
}
@@ -53,7 +53,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -61,7 +61,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -69,7 +69,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_host: &(T, Option<u16>),
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -77,7 +77,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_: &'a Path,
_: &str,
) -> Result<Cow<'a, Path>, deno_core::anyhow::Error> {
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
todo!()
}
}
@@ -95,6 +95,7 @@ fn main() {
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
let exts = vec![
// deno_telemetry::deno_telemetry::init_ops_and_esm(),
deno_webidl::deno_webidl::init_ops_and_esm(),
deno_url::deno_url::init_ops_and_esm(),
deno_console::deno_console::init_ops_and_esm(),
@@ -116,10 +117,10 @@ fn main() {
deno_core::snapshot::CreateSnapshotOptions {
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
startup_snapshot: None,
extension_transpiler: None,
extensions: exts,
with_runtime_cb: None,
skip_op_registration: false,
extension_transpiler: None,
},
None,
)
backend/windmill-worker/src/ansible_executor.rs
+7 -7
View File
@@ -433,6 +433,10 @@ fi
.args(cmd_args)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
ansible_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
start_child_process(ansible_cmd, ANSIBLE_PLAYBOOK_PATH.as_str()).await?
};
@@ -565,13 +569,9 @@ async fn create_file_resources(
get_resource_or_variable_content(client, &file_res.resource_path, job_id.to_string())
.await?;
let path = file_res.target_path.clone();
let validated_path = write_file_at_user_defined_location(
job_dir,
path.as_str(),
&r,
file_res.mode,
)
.map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?;
let validated_path =
write_file_at_user_defined_location(job_dir, path.as_str(), &r, file_res.mode)
.map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?;
nsjail_mounts.push(
define_nsjail_mount(job_dir, &validated_path)
backend/windmill-worker/src/bash_executor.rs
+229 -2
View File
@@ -1,14 +1,30 @@
use std::{collections::HashMap, fs, process::Stdio};
#[cfg(feature = "dind")]
use bollard::container::{
KillContainerOptions, RemoveContainerOptions, StatsOptions, StopContainerOptions,
};
#[cfg(feature = "dind")]
use futures::{stream, StreamExt, TryStreamExt};
use regex::Regex;
use serde_json::{json, value::RawValue};
use sqlx::types::Json;
use tokio::process::Command;
#[cfg(feature = "dind")]
use uuid::Uuid;
use windmill_common::{
error::Error,
jobs::QueuedJob,
worker::{to_raw_value, write_file},
};
#[cfg(feature = "dind")]
use windmill_common::DB;
#[cfg(feature = "dind")]
use windmill_common::error::to_anyhow;
use windmill_queue::{append_logs, CanceledBy};
lazy_static::lazy_static! {
@@ -22,6 +38,9 @@ lazy_static::lazy_static! {
static ref RE_POWERSHELL_IMPORTS: Regex = Regex::new(r#"^Import-Module\s+(?:-Name\s+)?"?([^-\s"]+)"?"#).unwrap();
}
#[cfg(feature = "dind")]
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::{
common::{
build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process,
@@ -54,8 +73,14 @@ pub async fn handle_bash_job(
worker_name: &str,
envs: HashMap<String, String>,
occupancy_metrics: &mut OccupancyMetrics,
_killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
) -> Result<Box<RawValue>, Error> {
let logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string();
let annotation = windmill_common::worker::BashAnnotations::parse(&content);
let mut logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string();
if annotation.docker {
logs1.push_str("docker mode\n");
}
append_logs(&job.id, &job.workspace_id, logs1, db).await;
write_file(job_dir, "main.sh", &format!("set -e\n{content}"))?;
@@ -193,6 +218,22 @@ exit $exit_status
)
.await?;
#[cfg(feature = "dind")]
if annotation.docker {
return handle_docker_job(
job.id,
&job.workspace_id,
db,
job.timeout,
mem_peak,
canceled_by,
worker_name,
occupancy_metrics,
_killpill_rx,
)
.await;
}
let result_json_path = format!("{job_dir}/result.json");
if let Ok(metadata) = tokio::fs::metadata(&result_json_path).await {
if metadata.len() > 0 {
@@ -214,6 +255,7 @@ exit $exit_status
.await?
.trim()
.to_string();
return Ok(to_raw_value(&json!(result)));
}
@@ -222,6 +264,190 @@ exit $exit_status
)))
}
#[cfg(feature = "dind")]
async fn handle_docker_job(
job_id: Uuid,
workspace_id: &str,
db: &DB,
job_timeout: Option<i32>,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
occupancy_metrics: &mut OccupancyMetrics,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
) -> Result<Box<RawValue>, Error> {
let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow)?;
let container_id = job_id.to_string();
let inspected = client.inspect_container(&container_id, None).await;
if inspected.is_err() || inspected.unwrap().state.is_none() {
return Ok(to_raw_value(&format!(
"Container not found at {job_id}, you must use --name and not --rm it"
)));
}
let wait_f = async {
let wait = client
.wait_container::<String>(&container_id, None)
.try_collect::<Vec<_>>()
.await
.map_err(|e| {
tracing::error!("Error waiting for container: {:?}", e);
anyhow::anyhow!("Error waiting for container")
})?;
let waited = wait.first().map(|x| x.status_code);
Ok(waited)
};
let ncontainer_id = container_id.to_string();
let w_id = workspace_id.to_string();
let j_id = job_id.clone();
let db2 = db.clone();
let (tx, mut rx) = tokio::sync::broadcast::channel::<()>(1);
let mut killpill_rx = killpill_rx.resubscribe();
let logs = tokio::spawn(async move {
let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow);
if let Ok(client) = client {
let mut log_stream = client.logs(
&ncontainer_id,
Some(bollard::container::LogsOptions {
follow: true,
stdout: true,
stderr: true,
tail: "all",
..Default::default()
}),
);
loop {
tokio::select! {
log = log_stream.next() => {
match log {
Some(Ok(log)) => {
append_logs(&j_id, w_id.clone(), log.to_string(), db2.clone()).await;
}
Some(Err(e)) => {
tracing::error!("Error getting logs: {:?}", e);
}
_ => {
tracing::error!("End of stream");
return
}
};
},
_ = killpill_rx.recv() => {
tracing::error!("killing container after receving killpill");
if let Err(e) = client
.stop_container(&ncontainer_id, Some(StopContainerOptions { t: 3 }))
.await
{
tracing::error!("Error stopping container: {:?}", e);
}
return
},
_ = rx.recv() => {
return
}
}
}
}
});
let mem_client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow);
let ncontainer_id = container_id.clone();
let result = run_future_with_polling_update_job_poller(
job_id,
job_timeout,
db,
mem_peak,
canceled_by,
wait_f,
worker_name,
workspace_id,
&mut Some(occupancy_metrics),
Box::pin(match mem_client {
Ok(client) => client
.stats(
&ncontainer_id,
Some(StatsOptions { stream: true, one_shot: false }),
)
.map(|x| {
x.map(|x| x.memory_stats.usage.map(|m| m / 1024).unwrap_or_default() as i32)
.unwrap_or_default()
})
.boxed(),
_ => stream::once(async { 0 }).boxed(),
}),
)
.await;
if let Err(e) = result {
if !logs.is_finished() {
let _ = tx.send(());
let _ = logs.await;
}
if container_is_alive(&client, &container_id).await {
kill_container(&client, &container_id, "SIGINT").await;
if container_is_alive(&client, &container_id).await {
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
if let Err(e) = client
.stop_container(&container_id, Some(StopContainerOptions { t: 3 }))
.await
{
tracing::error!("Error stopping container: {:?}", e);
}
}
}
return Err(e);
}
if let Err(e) = client
.remove_container(
&container_id,
Some(RemoveContainerOptions { force: true, ..Default::default() }),
)
.await
{
tracing::error!("Error removing container: {:?}", e);
}
let result = result.unwrap();
return Ok(to_raw_value(&json!(format!(
"Docker exit status: {}",
result
.map(|x| x.to_string())
.unwrap_or_else(|| "none".to_string())
))));
}
#[cfg(feature = "dind")]
async fn kill_container(client: &bollard::Docker, container_id: &str, signal: &str) {
if let Err(e) = client
.kill_container(&container_id, Some(KillContainerOptions { signal }))
.await
{
tracing::error!("Error killing container with signal {signal}: {:?}", e);
}
}
#[cfg(feature = "dind")]
async fn container_is_alive(client: &bollard::Docker, container_id: &str) -> bool {
let inspect = client.inspect_container(container_id, None).await;
if let Ok(inspect) = inspect {
let r = inspect
.state
.map(|x| x.running.unwrap_or_default())
.unwrap_or_default();
tracing::error!("Container {container_id} is alive: {r}");
r
} else {
false
}
}
fn raw_to_string(x: &str) -> String {
match serde_json::from_str::<serde_json::Value>(x) {
Ok(serde_json::Value::String(x)) => x,
@@ -522,7 +748,8 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"",
std::env::var("PATHEXT").unwrap_or_else(|_| {
String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL")
}),
);
)
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
}
cmd.spawn()?
backend/windmill-worker/src/bigquery_executor.rs
+1
View File
@@ -366,6 +366,7 @@ pub async fn do_bigquery(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/bun_executor.rs
+9
View File
@@ -1522,6 +1522,15 @@ pub async fn get_common_bun_proc_envs(base_internal_url: Option<&str>) -> HashMa
bun_envs.insert(String::from("NODE_PATH"), node_path.to_string());
}
#[cfg(windows)]
{
bun_envs.insert("SystemRoot".to_string(), crate::SYSTEM_ROOT.to_string());
bun_envs.insert(
"USERPROFILE".to_string(),
crate::USERPROFILE_ENV.to_string(),
);
}
return bun_envs;
}
backend/windmill-worker/src/common.rs
+5 -5
View File
@@ -866,11 +866,11 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error {
#[cfg(windows)]
let err_msg = "program not found";
if err
.to_string()
.contains(&err_msg)
{
return Error::InternalErr(format!("Executable {executable} not found on worker. PATH: {}", *PATH_ENV));
if err.to_string().contains(&err_msg) {
return Error::InternalErr(format!(
"Executable {executable} not found on worker. PATH: {}",
*PATH_ENV
));
}
return err;
}
backend/windmill-worker/src/deno_executor.rs
+9
View File
@@ -84,6 +84,15 @@ async fn get_common_deno_proc_envs(
if DENO_TLS_CA_STORE.len() > 0 {
deno_envs.insert(String::from("DENO_TLS_CA_STORE"), DENO_TLS_CA_STORE.clone());
}
#[cfg(windows)]
{
deno_envs.insert("SystemRoot".to_string(), crate::SYSTEM_ROOT.to_string());
deno_envs.insert(
"USERPROFILE".to_string(),
crate::USERPROFILE_ENV.to_string(),
);
}
return deno_envs;
}
backend/windmill-worker/src/go_executor.rs
+7
View File
@@ -194,6 +194,10 @@ func Run(req Req) (interface{{}}, error){{
.args(vec!["build", "main.go"])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
build_go_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
let build_go_process = start_child_process(build_go_cmd, GO_PATH.as_str()).await?;
handle_child(
&job.id,
@@ -293,6 +297,9 @@ func Run(req Req) (interface{{}}, error){{
run_go.env("GOPROXY", goproxy);
}
#[cfg(windows)]
run_go.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
run_go.stdout(Stdio::piped()).stderr(Stdio::piped());
start_child_process(run_go, compiled_executable_name).await?
};
backend/windmill-worker/src/graphql_executor.rs
+2 -1
View File
@@ -1,7 +1,7 @@
use std::collections::HashMap;
use anyhow::anyhow;
use futures::TryStreamExt;
use futures::{stream, TryStreamExt};
use serde_json::{json, value::RawValue};
use sqlx::types::Json;
use windmill_common::jobs::QueuedJob;
@@ -154,6 +154,7 @@ pub async fn do_graphql(
worker_name,
&job.workspace_id,
&mut Some(occupation_metrics),
Box::pin(stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/handle_child.rs
+20 -18
View File
@@ -87,7 +87,7 @@ async fn kill_process_tree(pid: Option<u32>) -> Result<(), String> {
/// - update the `last_line` and `logs` strings with the program output
/// - update "queue"."last_ping" every five seconds
/// - kill process if we exceed timeout or "queue"."canceled" is set
#[tracing::instrument(level = "trace", skip_all)]
#[tracing::instrument(name="run_subprocess", level = "info", skip_all, fields(otel.name = %child_name))]
pub async fn handle_child(
job_id: &Uuid,
db: &Pool<Postgres>,
@@ -126,7 +126,7 @@ pub async fn handle_child(
let (tx, rx) = broadcast::channel::<()>(3);
let mut rx2 = tx.subscribe();
let output = child_joined_output_stream(&mut child);
let output = child_joined_output_stream(&mut child, job_id.clone());
let job_id = job_id.clone();
@@ -137,7 +137,9 @@ pub async fn handle_child(
db,
mem_peak,
canceled_by_ref,
|| get_mem_peak(pid, nsjail),
Box::pin(stream::unfold((), move |_| async move {
Some((get_mem_peak(pid, nsjail).await, ()))
})),
worker,
w_id,
rx,
@@ -435,8 +437,6 @@ pub async fn handle_child(
}
}
async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
if pid.is_none() {
return -1;
@@ -488,7 +488,7 @@ async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
}
}
pub async fn run_future_with_polling_update_job_poller<Fut, T>(
pub async fn run_future_with_polling_update_job_poller<Fut, T, S>(
job_id: Uuid,
timeout: Option<i32>,
db: &DB,
@@ -498,9 +498,11 @@ pub async fn run_future_with_polling_update_job_poller<Fut, T>(
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
get_mem: S,
) -> error::Result<T>
where
Fut: Future<Output = anyhow::Result<T>>,
S: stream::Stream<Item = i32> + Unpin,
{
let (tx, rx) = broadcast::channel::<()>(3);
@@ -509,7 +511,7 @@ where
db,
mem_peak,
canceled_by_ref,
|| async { 0 },
get_mem,
worker_name,
w_id,
rx,
@@ -550,20 +552,19 @@ pub enum UpdateJobPollingExit {
AlreadyCompleted,
}
pub async fn update_job_poller<F, Fut>(
pub async fn update_job_poller<S>(
job_id: Uuid,
db: &DB,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
get_mem: F,
mut get_mem: S,
worker_name: &str,
w_id: &str,
mut rx: broadcast::Receiver<()>,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> UpdateJobPollingExit
where
F: Fn() -> Fut,
Fut: Future<Output = i32>,
S: stream::Stream<Item = i32> + Unpin,
{
let update_job_interval = Duration::from_millis(500);
@@ -608,7 +609,7 @@ where
.expect("update worker ping");
}
}
let current_mem = get_mem().await;
let current_mem = get_mem.next().await.unwrap_or(0);
if current_mem > *mem_peak {
*mem_peak = current_mem
}
@@ -679,6 +680,7 @@ where
/// builds a stream joining both stdout and stderr each read line by line
fn child_joined_output_stream(
child: &mut Child,
job_id: Uuid,
) -> impl stream::FusedStream<Item = io::Result<String>> {
let stderr = child
.stderr
@@ -692,24 +694,24 @@ fn child_joined_output_stream(
let stdout = BufReader::new(stdout).lines();
let stderr = BufReader::new(stderr).lines();
stream::select(lines_to_stream(stderr, true), lines_to_stream(stdout, false))
stream::select(
lines_to_stream(stderr, true, job_id.clone()),
lines_to_stream(stdout, false, job_id),
)
}
pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
mut lines: tokio::io::Lines<R>,
stderr: bool,
job_id: Uuid,
) -> impl futures::Stream<Item = io::Result<String>> {
stream::poll_fn(move |cx| {
std::pin::Pin::new(&mut lines)
.poll_next_line(cx)
.map(|result| {
process_streaming_log_lines(result, stderr)
})
.map(|result| process_streaming_log_lines(result, stderr, &job_id))
})
}
pub fn process_status(status: ExitStatus) -> error::Result<()> {
if status.success() {
Ok(())
backend/windmill-worker/src/job_logger_ee.rs
+1
View File
@@ -34,6 +34,7 @@ pub(crate) async fn default_disk_log_storage(
pub(crate) fn process_streaming_log_lines(
r: Result<Option<String>, io::Error>,
_stderr: bool,
_job_id: &Uuid,
) -> Option<Result<String, io::Error>> {
r.transpose()
}
backend/windmill-worker/src/js_eval.rs
+40 -49
View File
@@ -7,14 +7,7 @@
*/
#[cfg(feature = "deno_core")]
use std::{
borrow::Cow,
cell::RefCell,
env,
io::{self, BufReader},
path::PathBuf,
rc::Rc,
};
use std::{borrow::Cow, cell::RefCell, env, path::PathBuf, rc::Rc};
use std::{collections::HashMap, sync::Arc};
@@ -32,7 +25,7 @@ use deno_fetch::FetchPermissions;
#[cfg(feature = "deno_core")]
use deno_net::NetPermissions;
#[cfg(feature = "deno_core")]
use deno_tls::{rustls::RootCertStore, rustls_pemfile};
use deno_tls::rustls::RootCertStore;
#[cfg(feature = "deno_core")]
use deno_web::{BlobStore, TimersPermission};
#[cfg(feature = "deno_core")]
@@ -68,35 +61,35 @@ pub struct IdContext {
pub previous_id: String,
}
#[cfg(feature = "deno_core")]
pub struct ContainerRootCertStoreProvider {
root_cert_store: RootCertStore,
}
// #[cfg(feature = "deno_core")]
// pub struct ContainerRootCertStoreProvider {
// root_cert_store: RootCertStore,
// }
#[cfg(feature = "deno_core")]
impl ContainerRootCertStoreProvider {
fn new() -> ContainerRootCertStoreProvider {
return ContainerRootCertStoreProvider {
root_cert_store: deno_tls::create_default_root_cert_store(),
};
}
// #[cfg(feature = "deno_core")]
// impl ContainerRootCertStoreProvider {
// fn new() -> ContainerRootCertStoreProvider {
// return ContainerRootCertStoreProvider {
// root_cert_store: deno_tls::create_default_root_cert_store(),
// };
// }
fn add_certificate(&mut self, cert_path: String) -> io::Result<()> {
let cert_file = std::fs::File::open(cert_path)?;
let mut reader = BufReader::new(cert_file);
let pem_file = rustls_pemfile::certs(&mut reader).collect::<Result<Vec<_>, _>>()?;
// fn add_certificate(&mut self, cert_path: String) -> io::Result<()> {
// let cert_file = std::fs::File::open(cert_path)?;
// let mut reader = BufReader::new(cert_file);
// let pem_file = rustls_pemfile::certs(&mut reader).collect::<Result<Vec<_>, _>>()?;
self.root_cert_store.add_parsable_certificates(pem_file);
Ok(())
}
}
// self.root_cert_store.add_parsable_certificates(pem_file);
// Ok(())
// }
// }
#[cfg(feature = "deno_core")]
impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider {
fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> {
Ok(&self.root_cert_store)
}
}
// #[cfg(feature = "deno_core")]
// impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider {
// fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> {
// Ok(&self.root_cert_store)
// }
// }
#[cfg(feature = "deno_core")]
pub struct PermissionsContainer;
@@ -108,7 +101,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
_url: &deno_core::url::Url,
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
Ok(())
}
@@ -117,7 +110,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
p: &'a std::path::Path,
_api_name: &str,
) -> Result<Cow<'a, std::path::Path>, anyhow::Error> {
) -> Result<Cow<'a, std::path::Path>, deno_permissions::PermissionCheckError> {
Ok(Cow::Borrowed(p))
}
}
@@ -136,7 +129,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
Ok(PathBuf::from(p))
}
@@ -144,7 +137,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
Ok(PathBuf::from(p))
}
@@ -152,7 +145,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_host: &(T, Option<u16>),
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
Ok(())
}
@@ -160,7 +153,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
p: &'a std::path::Path,
_api_name: &str,
) -> Result<std::borrow::Cow<'a, std::path::Path>, AnyError> {
) -> Result<std::borrow::Cow<'a, std::path::Path>, deno_permissions::PermissionCheckError> {
Ok(Cow::Borrowed(p))
}
}
@@ -669,9 +662,12 @@ pub fn transpile_ts(expr: String) -> anyhow::Result<String> {
text: deno_core::ModuleCodeString::from(expr).into(),
})?;
Ok(parsed
.transpile(&Default::default(), &Default::default())?
.transpile(
&Default::default(),
&Default::default(),
&Default::default(),
)?
.into_source()
.into_string()?
.text)
}
@@ -814,13 +810,7 @@ pub async fn eval_fetch_timeout(
let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() };
let fetch_options = deno_fetch::Options {
root_cert_store_provider: if let Some(cert_path) = env::var("DENO_CERT").ok() {
let mut cert_store_provider = ContainerRootCertStoreProvider::new();
cert_store_provider.add_certificate(cert_path)?;
Some(Arc::new(cert_store_provider))
} else {
None
},
root_cert_store_provider: None,
user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()),
proxy: ann.proxy.map(|x| deno_tls::Proxy {
url: x.0,
@@ -930,6 +920,7 @@ pub async fn eval_fetch_timeout(
worker_name,
w_id,
&mut Some(occupation_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await
.map_err(|e| {
backend/windmill-worker/src/mssql_executor.rs
+1
View File
@@ -160,6 +160,7 @@ pub async fn do_mssql(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/mysql_executor.rs
+1
View File
@@ -298,6 +298,7 @@ pub async fn do_mysql(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/pg_executor.rs
+1
View File
@@ -355,6 +355,7 @@ pub async fn do_postgresql(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/python_executor.rs
+8 -5
View File
@@ -318,7 +318,7 @@ pub async fn uv_pip_compile(
if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
args.extend(["--cert", cert_path]);
}
tracing::debug!("uv args: {:?}", args);
tracing::error!("uv args: {:?}", args);
#[cfg(windows)]
let uv_cmd = "uv";
@@ -329,7 +329,7 @@ pub async fn uv_pip_compile(
let mut child_cmd = Command::new(uv_cmd);
child_cmd
.current_dir(job_dir)
.args(args)
.args(&args)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let child_process = start_child_process(child_cmd, uv_cmd).await?;
@@ -350,7 +350,7 @@ pub async fn uv_pip_compile(
occupancy_metrics,
)
.await
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed.\n\ncommand: {uv_cmd} {}\n\n{e:?}", args.join(" "))))?;
}
let path_lock = format!("{job_dir}/requirements.txt");
@@ -749,8 +749,10 @@ mount {{
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)]
python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
#[cfg(windows)] {
python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
python_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
}
start_child_process(python_cmd, PYTHON_PATH.as_str()).await?
};
@@ -1277,6 +1279,7 @@ async fn spawn_uv_install(
.envs(envs)
.envs(PROXY_ENVS.clone())
.env("SystemRoot", SYSTEM_ROOT.as_str())
.env("USERPROFILE", crate::USERPROFILE_ENV.as_str())
.env(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")),
backend/windmill-worker/src/result_processor.rs
+94 -27
View File
@@ -1,3 +1,6 @@
#[cfg(feature = "otel")]
use opentelemetry::trace::FutureExt;
use serde::Serialize;
use sqlx::{types::Json, Pool, Postgres};
use std::{
@@ -7,6 +10,9 @@ use std::{
Arc,
},
};
use tracing::{field, Instrument};
#[cfg(not(feature = "otel"))]
use windmill_common::otel_ee::FutureExt;
use uuid::Uuid;
@@ -14,6 +20,7 @@ use windmill_common::{
add_time,
error::{self, Error},
jobs::{JobKind, QueuedJob},
utils::WarnAfterExt,
worker::{to_raw_value, WORKER_GROUP},
DB,
};
@@ -84,7 +91,49 @@ pub fn start_background_processor(
JobKind::Dependencies | JobKind::FlowDependencies
);
handle_receive_completed_job(
let success = jc.success;
let span = tracing::span!(
tracing::Level::INFO,
"job_postprocessing",
job_id = %jc.job.id, root_job = field::Empty, workspace_id = %jc.job.workspace_id, worker = %worker_name,tag = %jc.job.tag,
// hostname = %hostname,
language = field::Empty,
script_path = field::Empty,
flow_step_id = field::Empty,
parent_job = field::Empty,
otel.name = field::Empty
);
let rj = if let Some(root_job) = jc.job.root_job {
root_job
} else {
jc.job.id
};
windmill_common::otel_ee::set_span_parent(&span, &rj);
if let Some(lg) = jc.job.language.as_ref() {
span.record("language", lg.as_str());
}
if let Some(step_id) = jc.job.flow_step_id.as_ref() {
span.record(
"otel.name",
format!("job_postprocessing {}", step_id).as_str(),
);
span.record("flow_step_id", step_id.as_str());
} else {
span.record("otel.name", "job postprocessing");
}
if let Some(parent_job) = jc.job.parent_job.as_ref() {
span.record("parent_job", parent_job.to_string().as_str());
}
if let Some(script_path) = jc.job.script_path.as_ref() {
span.record("script_path", script_path.as_str());
}
if let Some(root_job) = jc.job.root_job.as_ref() {
span.record("root_job", root_job.to_string().as_str());
}
let root_job = handle_receive_completed_job(
jc,
&base_internal_url,
&db,
@@ -95,8 +144,13 @@ pub fn start_background_processor(
#[cfg(feature = "benchmark")]
&mut bench,
)
.instrument(span)
.await;
if let Some(root_job) = root_job {
windmill_common::otel_ee::add_root_flow_job_to_otlp(&root_job, success);
}
if is_init_script_and_failure {
tracing::error!("init script errored, exiting");
killpill_tx.send(()).unwrap_or_default();
@@ -198,7 +252,11 @@ async fn send_job_completed(
token,
duration,
};
job_completed_tx.send(jc).await.expect("send job completed")
job_completed_tx
.send(jc)
.with_context(windmill_common::otel_ee::otel_ctx())
.await
.expect("send job completed")
}
pub async fn process_result(
@@ -270,6 +328,7 @@ pub async fn process_result(
token,
duration,
)
.with_context(windmill_common::otel_ee::otel_ctx())
.await;
Ok(true)
}
@@ -314,6 +373,7 @@ pub async fn process_result(
token,
duration,
)
.with_context(windmill_common::otel_ee::otel_ctx())
.await;
Ok(false)
}
@@ -329,7 +389,7 @@ pub async fn handle_receive_completed_job(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) {
) -> Option<Arc<QueuedJob>> {
let token = jc.token.clone();
let workspace = jc.job.workspace_id.clone();
let client = AuthedClient {
@@ -341,7 +401,7 @@ pub async fn handle_receive_completed_job(
let job = jc.job.clone();
let mem_peak = jc.mem_peak.clone();
let canceled_by = jc.canceled_by.clone();
if let Err(err) = process_completed_job(
match process_completed_job(
jc,
&client,
db,
@@ -354,26 +414,29 @@ pub async fn handle_receive_completed_job(
)
.await
{
handle_job_error(
db,
&client,
job.as_ref(),
mem_peak,
canceled_by,
err,
false,
same_worker_tx.clone(),
&worker_dir,
worker_name,
job_completed_tx,
#[cfg(feature = "benchmark")]
bench,
)
.await;
Err(err) => {
handle_job_error(
db,
&client,
job.as_ref(),
mem_peak,
canceled_by,
err,
false,
same_worker_tx.clone(),
&worker_dir,
worker_name,
job_completed_tx,
#[cfg(feature = "benchmark")]
bench,
)
.await;
None
}
Ok(r) => r,
}
}
#[tracing::instrument(name = "completed_job", level = "info", skip_all, fields(job_id = %job.id))]
pub async fn process_completed_job(
JobCompleted { job, result, mem_peak, success, cached_res_path, canceled_by, duration, .. }: JobCompleted,
client: &AuthedClient,
@@ -383,7 +446,7 @@ pub async fn process_completed_job(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> windmill_common::error::Result<()> {
) -> windmill_common::error::Result<Option<Arc<QueuedJob>>> {
if success {
// println!("bef completed job{:?}", SystemTime::now());
if let Some(cached_path) = cached_res_path {
@@ -413,8 +476,8 @@ pub async fn process_completed_job(
if is_flow_step {
if let Some(parent_job) = parent_job {
tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)");
update_flow_status_after_job_completion(
// tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)");
let r = update_flow_status_after_job_completion(
db,
client,
parent_job,
@@ -431,10 +494,12 @@ pub async fn process_completed_job(
#[cfg(feature = "benchmark")]
bench,
)
.warn_after_seconds(10)
.await?;
add_time!(bench, "updated flow status END");
return Ok(r);
}
}
add_time!(bench, "updated flow status END");
} else {
let result = add_completed_job_error(
db,
@@ -452,7 +517,7 @@ pub async fn process_completed_job(
if job.is_flow_step {
if let Some(parent_job) = job.parent_job {
tracing::error!(parent_flow = %parent_job, subflow = %job.id, "process completed job error, updating flow status");
update_flow_status_after_job_completion(
let r = update_flow_status_after_job_completion(
db,
client,
parent_job,
@@ -469,11 +534,13 @@ pub async fn process_completed_job(
#[cfg(feature = "benchmark")]
bench,
)
.warn_after_seconds(10)
.await?;
return Ok(r);
}
}
}
Ok(())
return Ok(None);
}
#[tracing::instrument(name = "job_error", level = "info", skip_all, fields(job_id = %job.id))]
backend/windmill-worker/src/rust_executor.rs
+5 -1
View File
@@ -209,6 +209,7 @@ pub async fn build_rust_crate(
"TMP",
std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()),
);
build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
}
let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?;
@@ -402,7 +403,10 @@ pub async fn handle_rust_job(
.stderr(Stdio::piped());
#[cfg(windows)]
run_rust.env("SystemRoot", SYSTEM_ROOT.as_str());
{
run_rust.env("SystemRoot", SYSTEM_ROOT.as_str());
run_rust.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
}
start_child_process(run_rust, compiled_executable_name).await?
};
backend/windmill-worker/src/snowflake_executor.rs
+1
View File
@@ -419,6 +419,7 @@ pub async fn do_snowflake(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
*mem_peak = (r.get().len() / 1000) as i32;
backend/windmill-worker/src/windmill-client.js
+6
View File
@@ -558,6 +558,9 @@ var $QueuedJob = {
"identity",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
],
},
schedule_path: {
@@ -697,6 +700,9 @@ var $CompletedJob = {
"identity",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
],
},
schedule_path: {
backend/windmill-worker/src/worker.rs
+147 -53
View File
@@ -6,9 +6,15 @@
* LICENSE-AGPL for a copy of the license.
*/
// #[cfg(feature = "otel")]
// use opentelemetry::{global, KeyValue};
use windmill_common::{
apps::AppScriptId,
auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET},
scripts::PREVIEW_IS_TAR_CODEBASE_HASH,
utils::WarnAfterExt,
worker::{
get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage, write_file,
ROOT_CACHE_DIR, TMP_DIR,
@@ -23,7 +29,7 @@ use const_format::concatcp;
#[cfg(feature = "prometheus")]
use prometheus::IntCounter;
use tracing::Instrument;
use tracing::{field, Instrument};
#[cfg(feature = "prometheus")]
use windmill_common::METRICS_DEBUG_ENABLED;
#[cfg(feature = "prometheus")]
@@ -86,30 +92,12 @@ use tokio::{
use rand::Rng;
use crate::{
ansible_executor::handle_ansible_job,
bash_executor::{handle_bash_job, handle_powershell_job},
bun_executor::handle_bun_job,
common::{
ansible_executor::handle_ansible_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{
build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args,
update_worker_ping_for_failed_init_script, OccupancyMetrics,
},
deno_executor::handle_deno_job,
go_executor::handle_go_job,
graphql_executor::do_graphql,
handle_child::SLOW_LOGS,
handle_job_error,
job_logger::NO_LOGS_AT_ALL,
js_eval::{eval_fetch_timeout, transpile_ts},
mysql_executor::do_mysql,
pg_executor::do_postgresql,
php_executor::handle_php_job,
python_executor::handle_python_job,
result_processor::{process_result, start_background_processor},
rust_executor::handle_rust_job,
worker_flow::{handle_flow, update_flow_status_in_progress, Step},
worker_lockfiles::{
}, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, handle_child::SLOW_LOGS, handle_job_error, job_logger::NO_LOGS_AT_ALL, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{process_result, start_background_processor}, rust_executor::handle_rust_job, worker_flow::{handle_flow, update_flow_status_in_progress, Step}, worker_lockfiles::{
handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job,
},
}
};
use backon::ConstantBuilder;
@@ -159,6 +147,7 @@ pub async fn create_token_for_owner_in_bg(
&email,
&job_id,
)
.warn_after_seconds(5)
.await
.expect("could not create job token");
*locked = token;
@@ -366,6 +355,7 @@ lazy_static::lazy_static! {
pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string());
pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new());
pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string());
pub static ref NODE_PATH: Option<String> = std::env::var("NODE_PATH").ok();
pub static ref TZ_ENV: String = std::env::var("TZ").unwrap_or_else(|_| String::new());
@@ -419,12 +409,12 @@ lazy_static::lazy_static! {
.and_then(|x| x.parse().ok())
.unwrap_or(false);
}
#[cfg(windows)]
lazy_static::lazy_static! {
pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string());
pub static ref USERPROFILE_ENV: String = std::env::var("USERPROFILE").unwrap_or_else(|_| "/tmp".to_string());
}
//only matter if CLOUD_HOSTED
@@ -716,7 +706,10 @@ fn add_outstanding_wait_time(
}.in_current_span());
}
#[tracing::instrument(name = "worker", level = "info", skip_all, fields(worker = %worker_name, hostname = %hostname))]
// struct WorkerMtrics {
// job_
// }
pub async fn run_worker(
db: &Pool<Postgres>,
hostname: &str,
@@ -732,6 +725,7 @@ pub async fn run_worker(
#[cfg(not(feature = "enterprise"))]
if !*DISABLE_NSJAIL {
tracing::warn!(
worker = %worker_name, hostname = %hostname,
"NSJAIL to sandbox process in untrusted environments is an enterprise feature but allowed to be used for testing purposes"
);
}
@@ -739,10 +733,10 @@ pub async fn run_worker(
let start_time = Instant::now();
let worker_dir = format!("{TMP_DIR}/{worker_name}");
tracing::debug!(worker_dir = %worker_dir, "Creating worker dir");
tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir");
if let Some(ref netrc) = *NETRC {
tracing::info!("Writing netrc at {}/.netrc", HOME_ENV.as_str());
tracing::info!(worker = %worker_name, hostname = %hostname, "Writing netrc at {}/.netrc", HOME_ENV.as_str());
write_file(&HOME_ENV, ".netrc", netrc).expect("could not write netrc");
}
@@ -957,6 +951,16 @@ pub async fn run_worker(
None
};
// let worker_resource = &[
// KeyValue::new("hostname", hostname.to_string()),
// KeyValue::new("worker", worker_name.to_string()),
// ];
// // Create a meter from the above MeterProvider.
// let meter = global::meter("windmill");
// let counter = meter.u64_counter("jobs.execution").build();
let mut occupancy_metrics = OccupancyMetrics::new(start_time);
let mut jobs_executed = 0;
@@ -1012,6 +1016,7 @@ pub async fn run_worker(
IS_READY.store(true, Ordering::Relaxed);
tracing::info!(
worker = %worker_name, hostname = %hostname,
"listening for jobs, WORKER_GROUP: {}, config: {:?}",
*WORKER_GROUP,
WORKER_CONFIG.read().await
@@ -1047,7 +1052,7 @@ pub async fn run_worker(
if i_worker == 1 {
if let Err(e) = queue_init_bash_maybe(db, same_worker_tx.clone(), &worker_name).await {
killpill_tx.send(()).unwrap_or_default();
tracing::error!("Error queuing init bash script for worker {worker_name}: {e:#}");
tracing::error!(worker = %worker_name, hostname = %hostname, "Error queuing init bash script for worker {worker_name}: {e:#}");
return;
}
}
@@ -1076,11 +1081,12 @@ pub async fn run_worker(
let mut last_suspend_first = Instant::now();
let mut killed_but_draining_same_worker_jobs = false;
let mut killpill_rx2 = killpill_rx.resubscribe();
loop {
#[cfg(feature = "enterprise")]
{
if let Ok(_) = killpill_rx.try_recv() {
tracing::info!("killpill received on worker waiting for valid key");
tracing::info!(worker = %worker_name, hostname = %hostname, "killpill received on worker waiting for valid key");
job_completed_tx
.0
.send(SendResult::Kill)
@@ -1092,6 +1098,7 @@ pub async fn run_worker(
if !valid_key {
tracing::error!(
worker = %worker_name, hostname = %hostname,
"Invalid license key, workers require a valid license key, sleeping for 30s waiting for valid key to be set"
);
tokio::time::sleep(Duration::from_secs(10)).await;
@@ -1105,7 +1112,7 @@ pub async fn run_worker(
#[cfg(feature = "prometheus")]
if let Some(wk) = worker_busy.as_ref() {
wk.set(0);
tracing::debug!("set worker busy to 0");
tracing::debug!(worker = %worker_name, hostname = %hostname, "set worker busy to 0");
}
occupancy_metrics.running_job_started_at = None;
@@ -1117,7 +1124,7 @@ pub async fn run_worker(
.try_into()
.unwrap(),
);
tracing::debug!("set uptime metric");
tracing::debug!(worker = %worker_name, hostname = %hostname, "set uptime metric");
}
if last_ping.elapsed().as_secs() > NUM_SECS_PING {
@@ -1161,15 +1168,19 @@ pub async fn run_worker(
)
.notify(|err, dur| {
tracing::error!(
worker = %worker_name, hostname = %hostname,
"retrying updating worker ping in {dur:#?}, err: {err:#?}"
);
})
.sleep(tokio::time::sleep)
.await {
tracing::error!("failed to update worker ping, exiting: {}", e);
tracing::error!(
worker = %worker_name, hostname = %hostname,
"failed to update worker ping, exiting: {}", e);
killpill_tx.send(()).unwrap_or_default();
}
tracing::info!(
worker = %worker_name, hostname = %hostname,
"ping update, memory: container={}MB, windmill={}MB",
memory_usage.unwrap_or_default() / (1024 * 1024),
wm_memory_usage.unwrap_or_default() / (1024 * 1024)
@@ -1181,16 +1192,18 @@ pub async fn run_worker(
if (jobs_executed as u32 + vacuum_shift) % VACUUM_PERIOD == 0 {
let db2 = db.clone();
let current_span = tracing::Span::current();
let worker_name = worker_name.clone();
let hostname = hostname.to_string();
tokio::task::spawn(
(async move {
tracing::info!("vacuuming queue and completed_job");
tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue");
if let Err(e) = sqlx::query!("VACUUM (skip_locked) queue")
.execute(&db2)
.await
{
tracing::error!("failed to vacuum queue: {}", e);
tracing::error!(worker = %worker_name, hostname = %hostname, "failed to vacuum queue: {}", e);
}
tracing::info!("vacuumed queue and completed_job");
tracing::info!(worker = %worker_name, hostname = %hostname, "vacuumed queue");
})
.instrument(current_span),
);
@@ -1226,6 +1239,7 @@ pub async fn run_worker(
if let Ok(same_worker_job) = same_worker_rx.try_recv() {
same_worker_queue_size.fetch_sub(1, Ordering::SeqCst);
tracing::debug!(
worker = %worker_name, hostname = %hostname,
"received {} from same worker channel",
same_worker_job.job_id
);
@@ -1238,6 +1252,7 @@ pub async fn run_worker(
.map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string()));
if r.is_err() && !same_worker_job.recoverable {
tracing::error!(
worker = %worker_name, hostname = %hostname,
"failed to fetch same_worker job on a non recoverable job, exiting"
);
job_completed_tx
@@ -1251,7 +1266,7 @@ pub async fn run_worker(
}
} else if let Ok(_) = killpill_rx.try_recv() {
if !killed_but_draining_same_worker_jobs {
tracing::info!("received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker);
tracing::info!(worker = %worker_name, hostname = %hostname, "received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker);
killed_but_draining_same_worker_jobs = true;
job_completed_tx
.0
@@ -1262,10 +1277,10 @@ pub async fn run_worker(
continue;
} else if killed_but_draining_same_worker_jobs {
if job_completed_processor_is_done.load(Ordering::SeqCst) {
tracing::info!("all running jobs have completed and all completed jobs have been fully processed, exiting");
tracing::info!(worker = %worker_name, hostname = %hostname, "all running jobs have completed and all completed jobs have been fully processed, exiting");
break;
} else {
tracing::info!("there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting");
tracing::info!(worker = %worker_name, hostname = %hostname, "there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting");
tokio::time::sleep(Duration::from_millis(200)).await;
continue;
}
@@ -1290,7 +1305,7 @@ pub async fn run_worker(
if !agent_mode && duration_pull_s > 0.5 {
let empty = job.as_ref().is_ok_and(|x| x.0.is_none());
tracing::warn!("pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
#[cfg(feature = "prometheus")]
if empty {
if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() {
@@ -1301,7 +1316,7 @@ pub async fn run_worker(
}
} else if !agent_mode && duration_pull_s > 0.1 {
let empty = job.as_ref().is_ok_and(|x| x.0.is_none());
tracing::warn!("pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
#[cfg(feature = "prometheus")]
if empty {
if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() {
@@ -1355,7 +1370,7 @@ pub async fn run_worker(
last_executed_job = None;
jobs_executed += 1;
tracing::debug!("started handling of job {}", job.id);
tracing::debug!(worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id);
if matches!(job.job_kind, JobKind::Script | JobKind::Preview) {
if !dedicated_workers.is_empty() {
@@ -1420,6 +1435,11 @@ pub async fn run_worker(
)
.await;
// counter.add(
// 1,
// worker_resource
// );
#[cfg(feature = "prometheus")]
let _timer = register_metric(
&WORKER_EXECUTION_DURATION,
@@ -1507,6 +1527,40 @@ pub async fn run_worker(
let PulledJob { job, raw_code, raw_lock, raw_flow } = job;
let arc_job = Arc::new(job);
add_time!(bench, "handle_queued_job START");
let span = tracing::span!(tracing::Level::INFO, "job",
job_id = %arc_job.id, root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, hostname = %hostname, tag = %arc_job.tag,
language = field::Empty,
script_path = field::Empty, flow_step_id = field::Empty, parent_job = field::Empty,
otel.name = field::Empty);
let rj = if let Some(root_job) = arc_job.root_job {
root_job
} else {
arc_job.id
};
if let Some(lg) = arc_job.language.as_ref() {
span.record("language", lg.as_str());
}
if let Some(step_id) = arc_job.flow_step_id.as_ref() {
span.record("otel.name", format!("job {}", step_id).as_str());
span.record("flow_step_id", step_id.as_str());
} else {
span.record("otel.name", "job");
}
if let Some(parent_job) = arc_job.parent_job.as_ref() {
span.record("parent_job", parent_job.to_string().as_str());
}
if let Some(script_path) = arc_job.script_path.as_ref() {
span.record("script_path", script_path.as_str());
}
if let Some(root_job) = arc_job.root_job.as_ref() {
span.record("root_job", root_job.to_string().as_str());
}
windmill_common::otel_ee::set_span_parent(&span, &rj);
// span.context().span().add_event_with_timestamp("job created".to_string(), arc_job.created_at.into(), vec![]);
match handle_queued_job(
arc_job.clone(),
raw_code,
@@ -1522,9 +1576,11 @@ pub async fn run_worker(
base_internal_url,
job_completed_tx.clone(),
&mut occupancy_metrics,
&mut killpill_rx2,
#[cfg(feature = "benchmark")]
&mut bench,
)
.instrument(span)
.await
{
Err(err) => {
@@ -1564,6 +1620,8 @@ pub async fn run_worker(
_ => {}
}
#[cfg(feature = "prometheus")]
if let Some(duration) = _timer.map(|x| x.stop_and_record()) {
register_metric(
@@ -1603,7 +1661,7 @@ pub async fn run_worker(
if let Some(secs) = *EXIT_AFTER_NO_JOB_FOR_SECS {
if let Some(lj) = last_executed_job {
if lj.elapsed().as_secs() > secs {
tracing::info!("no job for {} seconds, exiting", secs);
tracing::info!(worker = %worker_name, hostname = %hostname, "no job for {} seconds, exiting", secs);
break;
}
} else {
@@ -1634,12 +1692,12 @@ pub async fn run_worker(
});
}
Err(err) => {
tracing::error!("Failed to pull jobs: {}", err);
tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err);
}
};
}
tracing::info!("worker {} exiting", worker_name);
tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exiting", worker_name);
#[cfg(feature = "benchmark")]
{
@@ -1654,22 +1712,24 @@ pub async fn run_worker(
if has_dedicated_workers {
for handle in dedicated_handles {
if let Err(e) = handle.await {
tracing::error!("error in dedicated worker waiting for it to end: {:?}", e)
tracing::error!(worker = %worker_name, hostname = %hostname, "error in dedicated worker waiting for it to end: {:?}", e)
}
}
tracing::info!("all dedicated workers have exited");
tracing::info!(worker = %worker_name, hostname = %hostname, "all dedicated workers have exited");
}
drop(job_completed_tx);
tracing::info!("waiting for job_completed_processor to finish processing remaining jobs");
tracing::info!(worker = %worker_name, hostname = %hostname, "waiting for job_completed_processor to finish processing remaining jobs");
if let Err(e) = send_result.await {
tracing::error!("error in awaiting send_result process: {e:?}")
}
tracing::info!("worker {} exited", worker_name);
tracing::info!("number of jobs executed: {}", jobs_executed);
tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name);
tracing::info!(worker = %worker_name, hostname = %hostname, "number of jobs executed: {}", jobs_executed);
}
async fn queue_init_bash_maybe<'c>(
db: &Pool<Postgres>,
same_worker_tx: SameWorkerSender,
@@ -1794,7 +1854,6 @@ pub struct PreviousResult<'a> {
pub previous_result: Option<&'a RawValue>,
}
#[tracing::instrument(name = "job", level = "info", skip_all, fields(job_id = %job.id))]
async fn handle_queued_job(
job: Arc<QueuedJob>,
raw_code: Option<String>,
@@ -1810,8 +1869,12 @@ async fn handle_queued_job(
base_internal_url: &str,
job_completed_tx: JobCompletedSender,
occupancy_metrics: &mut OccupancyMetrics,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> windmill_common::error::Result<bool> {
// Extract the active span from the context
if job.canceled {
return Err(Error::JsonErr(canceled_job_to_result(&job)));
}
@@ -1823,7 +1886,9 @@ async fn handle_queued_job(
if job.parent_job.is_none() && job.created_by.starts_with("email-") {
let daily_count = sqlx::query!(
"SELECT value FROM metrics WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day' ORDER BY created_at DESC LIMIT 1"
).fetch_optional(db).await?.map(|x| serde_json::from_value::<i64>(x.value).unwrap_or(1));
).fetch_optional(db)
.warn_after_seconds(5)
.await?.map(|x| serde_json::from_value::<i64>(x.value).unwrap_or(1));
if let Some(count) = daily_count {
if count >= 100 {
@@ -1836,6 +1901,7 @@ async fn handle_queued_job(
serde_json::json!(count + 1)
)
.execute(db)
.warn_after_seconds(5)
.await?;
}
} else {
@@ -1843,6 +1909,7 @@ async fn handle_queued_job(
"INSERT INTO metrics (id, value) VALUES ('email_trigger_usage', to_jsonb(1))"
)
.execute(db)
.warn_after_seconds(5)
.await?;
}
}
@@ -1855,6 +1922,7 @@ async fn handle_queued_job(
.ok_or_else(|| Error::InternalErr(format!("expected parent job")))?,
job.id,
)
.warn_after_seconds(5)
.await?;
Some(r)
@@ -1867,6 +1935,7 @@ async fn handle_queued_job(
&job.workspace_id
)
.execute(db)
.warn_after_seconds(5)
.await {
tracing::error!("Could not update parent job started_at flow_status: {}", e);
}
@@ -1883,6 +1952,7 @@ async fn handle_queued_job(
job.workspace_id
)
.fetch_one(db)
.warn_after_seconds(5)
.await
.map(|record| (record.raw_code, record.raw_lock, record.raw_flow))
.unwrap_or_default(),
@@ -1923,6 +1993,7 @@ async fn handle_queued_job(
&job.parent_job.unwrap()
)
.fetch_one(db)
.warn_after_seconds(5)
.await
.map_err(|e| {
Error::InternalErr(format!(
@@ -1957,6 +2028,7 @@ async fn handle_queued_job(
&job.workspace_id,
&cached_res_path,
)
.warn_after_seconds(5)
.await;
if let Some(cached_resource_value) = cached_resource_value_maybe {
{
@@ -1995,6 +2067,7 @@ async fn handle_queued_job(
worker_dir,
job_completed_tx.0.clone(),
)
.warn_after_seconds(10)
.await?;
Ok(true)
} else {
@@ -2107,6 +2180,7 @@ async fn handle_queued_job(
&mut column_order,
&mut new_args,
occupancy_metrics,
killpill_rx,
)
.await;
occupancy_metrics.total_duration_of_running_jobs +=
@@ -2145,6 +2219,7 @@ async fn handle_queued_job(
}
}
pub fn build_envs(
envs: Option<Vec<String>>,
) -> windmill_common::error::Result<HashMap<String, String>> {
@@ -2250,6 +2325,8 @@ async fn handle_code_execution_job(
column_order: &mut Option<Vec<String>>,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
) -> error::Result<Box<RawValue>> {
let ContentReqLangEnvs {
content: inner_content,
@@ -2285,9 +2362,25 @@ async fn handle_code_execution_job(
.await?
}
JobKind::FlowScript => {
let (lockfile, content) = cache::flow::fetch_script(db, FlowNodeId(
job.script_hash.unwrap_or(ScriptHash(0)).0
)).await?;
let (lockfile, content) = cache::flow::fetch_script(
db,
FlowNodeId(job.script_hash.unwrap_or(ScriptHash(0)).0),
)
.await?;
ContentReqLangEnvs {
content,
lockfile,
language: job.language.to_owned(),
envs: None,
codebase: None,
}
}
JobKind::AppScript => {
let (lockfile, content) = cache::app::fetch_script(
db,
AppScriptId(job.script_hash.unwrap_or(ScriptHash(0)).0),
)
.await?;
ContentReqLangEnvs {
content,
lockfile,
@@ -2577,6 +2670,7 @@ mount {{
worker_name,
envs,
occupancy_metrics,
killpill_rx,
)
.await
}
backend/windmill-worker/src/worker_flow.rs
+59 -40
View File
@@ -41,6 +41,7 @@ use windmill_common::jobs::{
script_hash_to_tag_and_limits, script_path_to_payload, BranchResults, JobPayload, QueuedJob,
RawCode, ENTRYPOINT_OVERRIDE,
};
use windmill_common::utils::WarnAfterExt;
use windmill_common::worker::to_raw_value;
use windmill_common::{
error::{self, to_anyhow, Error},
@@ -76,45 +77,34 @@ pub async fn update_flow_status_after_job_completion(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> error::Result<()> {
) -> error::Result<Option<Arc<QueuedJob>>> {
// this is manual tailrecursion because async_recursion blows up the stack
// todo!();
potentially_crash_for_testing();
let mut rec = update_flow_status_after_job_completion_internal(
db,
client,
let mut rec = RecUpdateFlowStatusAfterJobCompletion {
flow,
job_id_for_status,
w_id,
job_id_for_status: job_id_for_status.clone(),
success,
result,
unrecoverable,
same_worker_tx.clone(),
worker_dir,
stop_early_override,
false,
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
bench,
)
.await?;
while let Some(nrec) = rec {
skip_error_handler: false,
};
let mut unrecoverable = unrecoverable;
loop {
potentially_crash_for_testing();
rec = match update_flow_status_after_job_completion_internal(
let nrec = match update_flow_status_after_job_completion_internal(
db,
client,
nrec.flow,
&nrec.job_id_for_status,
rec.flow,
&rec.job_id_for_status,
w_id,
nrec.success,
nrec.result,
false,
rec.success,
rec.result,
unrecoverable,
same_worker_tx.clone(),
worker_dir,
nrec.stop_early_override,
nrec.skip_error_handler,
rec.stop_early_override,
rec.skip_error_handler,
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
@@ -124,12 +114,12 @@ pub async fn update_flow_status_after_job_completion(
{
Ok(j) => j,
Err(e) => {
tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", nrec.flow,&nrec.job_id_for_status);
tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", rec.flow, &rec.job_id_for_status);
update_flow_status_after_job_completion_internal(
db,
client,
nrec.flow,
&nrec.job_id_for_status,
rec.flow,
&rec.job_id_for_status,
w_id,
false,
Arc::new(to_raw_value(&Json(&WrappedError {
@@ -138,8 +128,8 @@ pub async fn update_flow_status_after_job_completion(
true,
same_worker_tx.clone(),
worker_dir,
nrec.stop_early_override,
nrec.skip_error_handler,
rec.stop_early_override,
rec.skip_error_handler,
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
@@ -147,9 +137,33 @@ pub async fn update_flow_status_after_job_completion(
)
.await?
}
};
unrecoverable = false;
match nrec {
UpdateFlowStatusAfterJobCompletion::Done(job) => {
add_time!(bench, "update flow status internal END");
return Ok(Some(job));
}
UpdateFlowStatusAfterJobCompletion::Rec(nrec) => {
rec = nrec;
},
UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch => {
add_time!(bench, "update flow status internal END");
return Ok(None);
},
UpdateFlowStatusAfterJobCompletion::NotDone => {
add_time!(bench, "update flow status internal END");
return Ok(None);
}
}
}
Ok(())
}
pub enum UpdateFlowStatusAfterJobCompletion {
Rec(RecUpdateFlowStatusAfterJobCompletion),
Done(Arc<QueuedJob>),
NotDone,
NonLastParallelBranch,
}
pub struct RecUpdateFlowStatusAfterJobCompletion {
flow: uuid::Uuid,
@@ -187,7 +201,7 @@ pub async fn update_flow_status_after_job_completion_internal(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
) -> error::Result<UpdateFlowStatusAfterJobCompletion> {
add_time!(bench, "update flow status internal START");
let (
should_continue_flow,
@@ -602,7 +616,7 @@ pub async fn update_flow_status_after_job_completion_internal(
);
}
add_time!(bench, "non final parallel flow finished");
return Ok(None);
return Ok(UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch);
}
}
FlowStatusModule::InProgress {
@@ -1109,6 +1123,7 @@ pub async fn update_flow_status_after_job_completion_internal(
worker_dir,
job_completed_tx,
)
.warn_after_seconds(10)
.await
{
Err(err) => {
@@ -1146,7 +1161,7 @@ pub async fn update_flow_status_after_job_completion_internal(
if let Some(parent_job) = flow_job.parent_job {
tracing::info!(subflow_id = %flow_job.id, parent_id = %parent_job, "subflow is finished, updating parent flow status");
return Ok(Some(RecUpdateFlowStatusAfterJobCompletion {
return Ok(UpdateFlowStatusAfterJobCompletion::Rec(RecUpdateFlowStatusAfterJobCompletion {
flow: parent_job,
job_id_for_status: flow,
success: success && !is_failure_step,
@@ -1160,9 +1175,9 @@ pub async fn update_flow_status_after_job_completion_internal(
}));
}
}
Ok(None)
Ok(UpdateFlowStatusAfterJobCompletion::Done(flow_job))
} else {
Ok(None)
Ok(UpdateFlowStatusAfterJobCompletion::NotDone)
}
}
@@ -1290,7 +1305,7 @@ async fn compute_skip_branchall_failure<'c>(
// )))
// }
fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u16, Duration)> {
fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u32, Duration)> {
(status.fail_count <= MAX_RETRY_ATTEMPTS)
.then(|| &retry)
.and_then(|retry| retry.interval(status.fail_count, false))
@@ -1510,7 +1525,7 @@ pub async fn handle_flow(
let schedule_path = flow_job.schedule_path.as_ref().unwrap();
let schedule =
get_schedule_opt(&mut tx, &flow_job.workspace_id, schedule_path).await?;
get_schedule_opt(&mut tx, &flow_job.workspace_id, schedule_path).warn_after_seconds(5).await?;
tx.commit().await?;
@@ -1522,6 +1537,7 @@ pub async fn handle_flow(
flow_job.script_path.as_ref().unwrap(),
&flow_job.workspace_id,
)
.warn_after_seconds(5)
.await
{
match err {
@@ -1549,6 +1565,7 @@ pub async fn handle_flow(
worker_dir,
job_completed_tx,
)
.warn_after_seconds(10)
.await?;
Ok(())
}
@@ -2133,6 +2150,7 @@ async fn push_next_flow_job(
.bind(status.step)
.bind(json!(status.retry.failed_jobs))
.execute(db)
.warn_after_seconds(2)
.await
.context("update flow retry")?;
@@ -2558,6 +2576,7 @@ async fn push_next_flow_job(
new_job_priority_override,
job_perms.as_ref(),
)
.warn_after_seconds(2)
.await?;
tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushed next flow job: {uuid}");
@@ -2770,7 +2789,7 @@ async fn push_next_flow_job(
.execute(&mut *tx)
.await?;
tx.commit().await?;
tx.commit().warn_after_seconds(3).await?;
tracing::info!(id = %flow_job.id, root_id = %job_root, "all next flow jobs pushed: {uuids:?}");
if continue_on_same_worker {
backend/windmill-worker/src/worker_lockfiles.rs
+195 -60
View File
@@ -4,6 +4,7 @@ use std::path::{Component, Path, PathBuf};
use async_recursion::async_recursion;
use serde_json::value::RawValue;
use serde_json::{json, Value};
use sha2::Digest;
use sqlx::types::Json;
use uuid::Uuid;
use windmill_common::error::Error;
@@ -14,6 +15,7 @@ use windmill_common::jobs::JobPayload;
use windmill_common::scripts::ScriptHash;
use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, PythonAnnotations};
use windmill_common::{
apps::AppScriptId,
error::{self, to_anyhow},
flows::{add_virtual_items_if_necessary, FlowValue},
jobs::QueuedJob,
@@ -606,9 +608,8 @@ pub async fn handle_flow_dependency_job(
occupancy_metrics,
)
.await?;
let new_flow_value = sqlx::types::Json(
serde_json::value::to_raw_value(&flow).map_err(to_anyhow)?
);
let new_flow_value =
sqlx::types::Json(serde_json::value::to_raw_value(&flow).map_err(to_anyhow)?);
// Re-check cancelation to ensure we don't accidentially override a flow.
if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id)
@@ -648,11 +649,20 @@ pub async fn handle_flow_dependency_job(
// Compute a lite version of the flow value (`RawScript` => `FlowScript`).
let mut value_lite = flow.clone();
tx = reduce(tx, &mut value_lite.modules, &job_path, &job.workspace_id, flow.failure_module.as_ref(), flow.same_worker).await?;
tx = reduce_flow(
tx,
&mut value_lite.modules,
&job_path,
&job.workspace_id,
flow.failure_module.as_ref(),
flow.same_worker,
)
.await?;
sqlx::query!(
"INSERT INTO flow_version_lite (id, value) VALUES ($1, $2)
ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value",
version, sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json<Box<RawValue>>,
version,
sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json<Box<RawValue>>,
)
.execute(db)
.await?;
@@ -829,8 +839,12 @@ async fn lock_modules<'c>(
occupancy_metrics,
))
.await?;
e.value =
FlowModuleValue::WhileloopFlow { modules: nmodules, modules_node, skip_failures }.into()
e.value = FlowModuleValue::WhileloopFlow {
modules: nmodules,
modules_node,
skip_failures,
}
.into()
}
FlowModuleValue::BranchOne { branches, default, default_node } => {
let mut nbranches = vec![];
@@ -878,8 +892,12 @@ async fn lock_modules<'c>(
occupancy_metrics,
))
.await?;
e.value = FlowModuleValue::BranchOne { branches: nbranches, default: ndefault, default_node }
.into();
e.value = FlowModuleValue::BranchOne {
branches: nbranches,
default: ndefault,
default_node,
}
.into();
}
_ => (),
};
@@ -1009,44 +1027,68 @@ async fn insert_flow_node<'c>(
flow: Option<&Json<Box<RawValue>>>,
) -> Result<(sqlx::Transaction<'c, sqlx::Postgres>, FlowNodeId)> {
let hash = {
use std::hash::{DefaultHasher, Hasher, Hash};
let mut hasher = DefaultHasher::new();
code.hash(&mut hasher);
lock.hash(&mut hasher);
flow.inspect(|flow| flow.get().hash(&mut hasher));
hasher.finish() as i64
let mut hasher = sha2::Sha256::new();
hasher.update(code.unwrap_or(&Default::default()));
hasher.update(lock.unwrap_or(&Default::default()));
hasher.update(flow.unwrap_or(&Default::default()).get());
format!("{:x}", hasher.finalize())
};
// Insert the flow node if it doesn't exist.
let id = sqlx::query_scalar!(
r#"
WITH existing AS (
SELECT id FROM flow_node
WHERE hash = $1 AND path = $2 AND workspace_id = $3
AND (code IS NOT DISTINCT FROM $4)
AND (lock IS NOT DISTINCT FROM $5)
AND (flow IS NOT DISTINCT FROM $6)
LIMIT 1
),
inserted AS (
INSERT INTO flow_node (hash, path, workspace_id, code, lock, flow)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT DO NOTHING
RETURNING id
)
SELECT id FROM existing
UNION ALL
SELECT id FROM inserted
INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id
RETURNING id
"#,
hash, path, workspace_id, code, lock, flow as Option<&Json<Box<RawValue>>>
path,
workspace_id,
hash,
lock,
code,
flow as Option<&Json<Box<RawValue>>>
)
.fetch_one(&mut *tx)
.await?
.ok_or(error::Error::InternalErr("Failed to cache".to_string()))?;
.await?;
Ok((tx, FlowNodeId(id)))
}
async fn insert_app_script(
db: &sqlx::Pool<sqlx::Postgres>,
app: i64,
code: String,
lock: Option<String>,
) -> Result<AppScriptId> {
let code_sha256 = format!("{:x}", sha2::Sha256::digest(&code));
let hash = {
let mut hasher = sha2::Sha256::new();
hasher.update(app.to_le_bytes());
hasher.update(&code_sha256);
hasher.update(lock.as_ref().unwrap_or(&Default::default()));
format!("{:x}", hasher.finalize())
};
// Insert the app script if it doesn't exist.
sqlx::query_scalar!(
r#"
INSERT INTO app_script (app, hash, lock, code, code_sha256)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id
RETURNING id
"#,
app,
hash,
lock,
code,
code_sha256
)
.fetch_one(db)
.await
.map(AppScriptId)
.map_err(Into::into)
}
async fn insert_flow_modules<'c>(
mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
path: &str,
@@ -1056,7 +1098,15 @@ async fn insert_flow_modules<'c>(
modules: &mut Vec<FlowModule>,
modules_node: &mut Option<FlowNodeId>,
) -> Result<sqlx::Transaction<'c, sqlx::Postgres>> {
tx = Box::pin(reduce(tx, modules, path, workspace_id, failure_module, same_worker)).await?;
tx = Box::pin(reduce_flow(
tx,
modules,
path,
workspace_id,
failure_module,
same_worker,
))
.await?;
add_virtual_items_if_necessary(modules);
if modules.is_empty() || crate::worker_flow::is_simple_modules(modules, failure_module) {
return Ok(tx);
@@ -1073,14 +1123,14 @@ async fn insert_flow_modules<'c>(
failure_module: failure_module.cloned(),
same_worker,
..Default::default()
})))
}))),
)
.await?;
*modules_node = Some(id);
Ok(tx)
}
async fn reduce<'c>(
async fn reduce_flow<'c>(
mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
modules: &mut Vec<FlowModule>,
path: &str,
@@ -1090,8 +1140,13 @@ async fn reduce<'c>(
) -> Result<sqlx::Transaction<'c, sqlx::Postgres>> {
use FlowModuleValue::*;
for module in &mut *modules {
let mut val = serde_json::from_str::<FlowModuleValue>(module.value.get())
.map_err(|err| Error::InternalErr(format!("reduce: Failed to parse flow module value: {}", err)))?;
let mut val =
serde_json::from_str::<FlowModuleValue>(module.value.get()).map_err(|err| {
Error::InternalErr(format!(
"reduce_flow: Failed to parse flow module value: {}",
err
))
})?;
match &mut val {
RawScript { .. } => {
// In order to avoid an unnecessary `.clone()` of `val`, take ownership of it's content
@@ -1107,9 +1162,14 @@ async fn reduce<'c>(
concurrency_time_window_s,
is_trigger,
..
} = std::mem::replace(&mut val, Identity) else { unreachable!() };
} = std::mem::replace(&mut val, Identity)
else {
unreachable!()
};
let id;
(tx, id) = insert_flow_node(tx, path, workspace_id, Some(&content), lock.as_ref(), None).await?;
(tx, id) =
insert_flow_node(tx, path, workspace_id, Some(&content), lock.as_ref(), None)
.await?;
val = FlowScript {
input_transforms,
id,
@@ -1120,32 +1180,56 @@ async fn reduce<'c>(
concurrency_time_window_s,
is_trigger,
};
},
}
ForloopFlow { modules, modules_node, .. }
| WhileloopFlow { modules, modules_node, .. } => {
| WhileloopFlow { modules, modules_node, .. } => {
tx = insert_flow_modules(
tx, path, workspace_id, failure_module, same_worker,
modules, modules_node
).await?;
tx,
path,
workspace_id,
failure_module,
same_worker,
modules,
modules_node,
)
.await?;
}
BranchOne { branches, default, default_node, .. } => {
for branch in branches.iter_mut() {
tx = insert_flow_modules(
tx, path, workspace_id, failure_module, same_worker,
&mut branch.modules, &mut branch.modules_node
).await?;
tx,
path,
workspace_id,
failure_module,
same_worker,
&mut branch.modules,
&mut branch.modules_node,
)
.await?;
}
tx = insert_flow_modules(
tx, path, workspace_id, failure_module, same_worker,
default, default_node
).await?;
tx,
path,
workspace_id,
failure_module,
same_worker,
default,
default_node,
)
.await?;
}
BranchAll { branches, .. } => {
for branch in branches.iter_mut() {
tx = insert_flow_modules(
tx, path, workspace_id, failure_module, same_worker,
&mut branch.modules, &mut branch.modules_node
).await?;
tx,
path,
workspace_id,
failure_module,
same_worker,
&mut branch.modules,
&mut branch.modules_node,
)
.await?;
}
}
_ => {}
@@ -1155,6 +1239,41 @@ async fn reduce<'c>(
Ok(tx)
}
async fn reduce_app(db: &sqlx::Pool<sqlx::Postgres>, value: &mut Value, app: i64) -> Result<()> {
match value {
Value::Object(object) => {
if let Some(Value::Object(script)) = object.get_mut("inlineScript") {
// replace `content` with an empty string:
let Some(Value::String(code)) = script.get_mut("content").map(std::mem::take)
else {
return Err(error::Error::InternalErr(
"Missing `content` in inlineScript".to_string(),
));
};
// remove `lock`:
let lock = script.remove("lock").and_then(|x| match x {
Value::String(s) => Some(s),
_ => None,
});
let id = insert_app_script(db, app, code, lock).await?;
// insert the `id` into the `script` object:
script.insert("id".to_string(), json!(id.0));
} else {
for (_, value) in object {
Box::pin(reduce_app(db, value, app)).await?;
}
}
}
Value::Array(array) => {
for value in array {
Box::pin(reduce_app(db, value, app)).await?;
}
}
_ => {}
}
Ok(())
}
fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool {
if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative {
let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content);
@@ -1340,11 +1459,12 @@ pub async fn handle_app_dependency_job(
.clone()
.ok_or_else(|| Error::InternalErr("App Dependency requires script hash".to_owned()))?
.0;
let value = sqlx::query_scalar!("SELECT value FROM app_version WHERE id = $1", id)
let record = sqlx::query!("SELECT app_id, value FROM app_version WHERE id = $1", id)
.fetch_optional(db)
.await?;
.await?
.map(|record| (record.app_id, record.value));
if let Some(value) = value {
if let Some((app_id, value)) = record {
let value = lock_modules_app(
value,
job,
@@ -1361,6 +1481,21 @@ pub async fn handle_app_dependency_job(
)
.await?;
// Compute a lite version of the app value (w/ `inlineScript.{lock,code}`).
let mut value_lite = value.clone();
reduce_app(db, &mut value_lite, app_id).await?;
if let Value::Object(object) = &mut value_lite {
object.insert("version".to_string(), json!(id));
}
sqlx::query!(
"INSERT INTO app_version_lite (id, value) VALUES ($1, $2)
ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value",
id,
sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json<Box<RawValue>>,
)
.execute(db)
.await?;
// Re-check cancelation to ensure we don't accidentially override an app.
if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id)
.fetch_optional(db)
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.434.2";
export const VERSION = "v1.435.2";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/main.ts
+1 -1
View File
@@ -60,7 +60,7 @@ export {
// }
// });
export const VERSION = "1.434.2";
export const VERSION = "1.435.2";
const command = new Command()
.name("wmill")
docker-compose.yml
-6
View File
@@ -129,12 +129,6 @@ services:
- PORT=8001
- DATABASE_URL=${DATABASE_URL}
- MODE=indexer
- TANTIVY_MAX_INDEXED_JOB_LOG_SIZE__MB=1 # job logs bigger than this will be truncated before indexing
- TANTIVY_S3_BACKUP_PERIOD__S=3600 # how often to backup the index into object storage
- TANTIVY_INDEX_WRITER_MEMORY_BUDGET__MB=100 # higher budget for higher indexing throughput
- TANTIVY_REFRESH_INDEX_PERIOD__S=300 #how often to start indexing new jobs
- TANTIVY_DOC_COMMIT_MAX_BATCH_SIZE=100000 #how many documents to batch in one commit
- TANTIVY_SHOW_MEMORY_EVERY=10000 #log memory usage and progress every so many documents indexed
depends_on:
db:
condition: service_healthy
docker/DockerfileSlim
+1 -1
View File
@@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
docker/DockerfileSlimEe
+1 -1
View File
@@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
frontend/package-lock.json
Generated
+68 -64
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.434.2",
"version": "1.435.2",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.434.2",
"version": "1.435.2",
"license": "AGPL-3.0",
"dependencies": {
"@anthropic-ai/sdk": "^0.32.1",
@@ -87,10 +87,10 @@
"@hey-api/openapi-ts": "^0.43.0",
"@playwright/test": "^1.34.3",
"@rgossiaux/svelte-headlessui": "^2.0.0",
"@sveltejs/adapter-static": "^3.0.0",
"@sveltejs/kit": "^2.0.0",
"@sveltejs/adapter-static": "^3.0.6",
"@sveltejs/kit": "^2.9.0",
"@sveltejs/package": "^2.2.2",
"@sveltejs/vite-plugin-svelte": "^3.0.0",
"@sveltejs/vite-plugin-svelte": "^3.1.2",
"@tailwindcss/forms": "^0.5.3",
"@tailwindcss/typography": "^0.5.8",
"@types/d3": "^7.4.0",
@@ -130,7 +130,7 @@
"tailwindcss": "^3.4.1",
"tslib": "^2.6.1",
"typescript": "^5.1.3",
"vite": "^5",
"vite": "^5.4.11",
"vite-plugin-circular-dependency": "^0.2.1",
"vite-plugin-mkcert": "^1.17.5",
"yootils": "^0.3.1"
@@ -3584,9 +3584,9 @@
}
},
"node_modules/@jridgewell/sourcemap-codec": {
"version": "1.4.15",
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz",
"integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg=="
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz",
"integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ=="
},
"node_modules/@jridgewell/trace-mapping": {
"version": "0.3.20",
@@ -3897,10 +3897,11 @@
}
},
"node_modules/@polka/url": {
"version": "1.0.0-next.24",
"resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.24.tgz",
"integrity": "sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ==",
"dev": true
"version": "1.0.0-next.28",
"resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.28.tgz",
"integrity": "sha512-8LduaNlMZGwdZ6qWrKlfa+2M4gahzFkprZiAt2TF8uS0qQgBizKXpXURqvTJ4WtmupWxaLqjRb2UCTe72mu+Aw==",
"dev": true,
"license": "MIT"
},
"node_modules/@popperjs/core": {
"version": "2.11.8",
@@ -4214,32 +4215,33 @@
}
},
"node_modules/@sveltejs/adapter-static": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.1.tgz",
"integrity": "sha512-6lMvf7xYEJ+oGeR5L8DFJJrowkefTK6ZgA4JiMqoClMkKq0s6yvsd3FZfCFvX1fQ0tpCD7fkuRVHsnUVgsHyNg==",
"version": "3.0.6",
"resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.6.tgz",
"integrity": "sha512-MGJcesnJWj7FxDcB/GbrdYD3q24Uk0PIL4QIX149ku+hlJuj//nxUbb0HxUTpjkecWfHjVveSUnUaQWnPRXlpg==",
"dev": true,
"peerDependencies": {
"@sveltejs/kit": "^2.0.0"
}
},
"node_modules/@sveltejs/kit": {
"version": "2.5.0",
"resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.5.0.tgz",
"integrity": "sha512-1uyXvzC2Lu1FZa30T4y5jUAC21R309ZMRG0TPt+PPPbNUoDpy8zSmSNVWYaBWxYDqLGQ5oPNWvjvvF2IjJ1jmA==",
"version": "2.9.0",
"resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.9.0.tgz",
"integrity": "sha512-W3E7ed3ChB6kPqRs2H7tcHp+Z7oiTFC6m+lLyAQQuyXeqw6LdNuuwEUla+5VM0OGgqQD+cYD6+7Xq80vVm17Vg==",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
"dependencies": {
"@types/cookie": "^0.6.0",
"cookie": "^0.6.0",
"devalue": "^4.3.2",
"esm-env": "^1.0.0",
"import-meta-resolve": "^4.0.0",
"devalue": "^5.1.0",
"esm-env": "^1.2.1",
"import-meta-resolve": "^4.1.0",
"kleur": "^4.1.5",
"magic-string": "^0.30.5",
"mrmime": "^2.0.0",
"sade": "^1.8.1",
"set-cookie-parser": "^2.6.0",
"sirv": "^2.0.4",
"sirv": "^3.0.0",
"tiny-glob": "^0.2.9"
},
"bin": {
@@ -4249,9 +4251,9 @@
"node": ">=18.13"
},
"peerDependencies": {
"@sveltejs/vite-plugin-svelte": "^3.0.0",
"@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0",
"svelte": "^4.0.0 || ^5.0.0-next.0",
"vite": "^5.0.3"
"vite": "^5.0.3 || ^6.0.0"
}
},
"node_modules/@sveltejs/package": {
@@ -4277,17 +4279,17 @@
}
},
"node_modules/@sveltejs/vite-plugin-svelte": {
"version": "3.0.2",
"resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-3.0.2.tgz",
"integrity": "sha512-MpmF/cju2HqUls50WyTHQBZUV3ovV/Uk8k66AN2gwHogNAG8wnW8xtZDhzNBsFJJuvmq1qnzA5kE7YfMJNFv2Q==",
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-3.1.2.tgz",
"integrity": "sha512-Txsm1tJvtiYeLUVRNqxZGKR/mI+CzuIQuc2gn+YCs9rMTowpNZ2Nqt53JdL8KF9bLhAf2ruR/dr9eZCwdTriRA==",
"dev": true,
"dependencies": {
"@sveltejs/vite-plugin-svelte-inspector": "^2.0.0",
"@sveltejs/vite-plugin-svelte-inspector": "^2.1.0",
"debug": "^4.3.4",
"deepmerge": "^4.3.1",
"kleur": "^4.1.5",
"magic-string": "^0.30.5",
"svelte-hmr": "^0.15.3",
"magic-string": "^0.30.10",
"svelte-hmr": "^0.16.0",
"vitefu": "^0.2.5"
},
"engines": {
@@ -4299,9 +4301,9 @@
}
},
"node_modules/@sveltejs/vite-plugin-svelte-inspector": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-2.0.0.tgz",
"integrity": "sha512-gjr9ZFg1BSlIpfZ4PRewigrvYmHWbDrq2uvvPB1AmTWKuM+dI1JXQSUu2pIrYLb/QncyiIGkFDFKTwJ0XqQZZg==",
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-2.1.0.tgz",
"integrity": "sha512-9QX28IymvBlSCqsCll5t0kQVxipsfhFFL+L2t3nTWfXnddYwxBuAEtTtlaVQpRz9c37BhJjltSeY4AJSC03SSg==",
"dev": true,
"dependencies": {
"debug": "^4.3.4"
@@ -6434,10 +6436,11 @@
}
},
"node_modules/devalue": {
"version": "4.3.2",
"resolved": "https://registry.npmjs.org/devalue/-/devalue-4.3.2.tgz",
"integrity": "sha512-KqFl6pOgOW+Y6wJgu80rHpo2/3H07vr8ntR9rkkFIRETewbf5GaYYcakYfiKz89K+sLsuPkQIZaXDMjUObZwWg==",
"dev": true
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/devalue/-/devalue-5.1.1.tgz",
"integrity": "sha512-maua5KUiapvEwiEAe+XnlZ3Rh0GD+qI1J/nb9vrJc3muPXvcF/8gXYTWF76+5DAqHyDUtOIImEuo0YKE9mshVw==",
"dev": true,
"license": "MIT"
},
"node_modules/devlop": {
"version": "1.1.0",
@@ -6990,9 +6993,10 @@
}
},
"node_modules/esm-env": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.0.0.tgz",
"integrity": "sha512-Cf6VksWPsTuW01vU9Mk/3vRue91Zevka5SjyNf3nEpokFRuqt/KjUQoGAwq9qMmhpLTHmXzSIrFRw8zxWzmFBA=="
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.1.tgz",
"integrity": "sha512-U9JedYYjCnadUlXk7e1Kr+aENQhtUaoaV9+gZm1T8LC/YBAPJx3NSPIAurFOC0U5vrdSevnUJS2/wUVxGwPhng==",
"license": "MIT"
},
"node_modules/esm-env-robust": {
"version": "0.0.3",
@@ -7932,10 +7936,11 @@
}
},
"node_modules/import-meta-resolve": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.0.0.tgz",
"integrity": "sha512-okYUR7ZQPH+efeuMJGlq4f8ubUgO50kByRPyt/Cy1Io4PSRsPjxME+YlVaCOx+NIToW7hCsZNFJyTPFFKepRSA==",
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz",
"integrity": "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==",
"dev": true,
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
@@ -8608,14 +8613,11 @@
}
},
"node_modules/magic-string": {
"version": "0.30.5",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.5.tgz",
"integrity": "sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==",
"version": "0.30.14",
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.14.tgz",
"integrity": "sha512-5c99P1WKTed11ZC0HMJOj6CDIue6F8ySu+bJL+85q1zBEIY8IklrJ1eiKC2NDRh3Ct3FcvmJPyQHb9erXMTJNw==",
"dependencies": {
"@jridgewell/sourcemap-codec": "^1.4.15"
},
"engines": {
"node": ">=12"
"@jridgewell/sourcemap-codec": "^1.5.0"
}
},
"node_modules/map-obj": {
@@ -9781,6 +9783,7 @@
"resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.0.tgz",
"integrity": "sha512-eu38+hdgojoyq63s+yTpN4XMBdt5l8HhMhc4VKLO9KM5caLIBvUm4thi7fFaxyTmCKeNnXZ5pAlBwCUnhA09uw==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=10"
}
@@ -11960,17 +11963,18 @@
"dev": true
},
"node_modules/sirv": {
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/sirv/-/sirv-2.0.4.tgz",
"integrity": "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==",
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.0.tgz",
"integrity": "sha512-BPwJGUeDaDCHihkORDchNyyTvWFhcusy1XMmhEVTQTwGeybFbp8YEmB+njbPnth1FibULBSBVwCQni25XlCUDg==",
"dev": true,
"license": "MIT",
"dependencies": {
"@polka/url": "^1.0.0-next.24",
"mrmime": "^2.0.0",
"totalist": "^3.0.0"
},
"engines": {
"node": ">= 10"
"node": ">=18"
}
},
"node_modules/slash": {
@@ -12642,9 +12646,9 @@
}
},
"node_modules/svelte-hmr": {
"version": "0.15.3",
"resolved": "https://registry.npmjs.org/svelte-hmr/-/svelte-hmr-0.15.3.tgz",
"integrity": "sha512-41snaPswvSf8TJUhlkoJBekRrABDXDMdpNpT2tfHIv4JuhgvHqLMhEPGtaQn0BmbNSTkuz2Ed20DF2eHw0SmBQ==",
"version": "0.16.0",
"resolved": "https://registry.npmjs.org/svelte-hmr/-/svelte-hmr-0.16.0.tgz",
"integrity": "sha512-Gyc7cOS3VJzLlfj7wKS0ZnzDVdv3Pn2IuVeJPk9m2skfhcu5bq3wtIZyQGggr7/Iim5rH5cncyQft/kRLupcnA==",
"dev": true,
"engines": {
"node": "^12.20 || ^14.13.1 || >= 16"
@@ -13064,6 +13068,7 @@
"resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz",
"integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==",
"dev": true,
"license": "MIT",
"engines": {
"node": ">=6"
}
@@ -13395,15 +13400,14 @@
}
},
"node_modules/vite": {
"version": "5.4.0",
"resolved": "https://registry.npmjs.org/vite/-/vite-5.4.0.tgz",
"integrity": "sha512-5xokfMX0PIiwCMCMb9ZJcMyh5wbBun0zUzKib+L65vAZ8GY9ePZMXxFrHbr/Kyll2+LSCY7xtERPpxkBDKngwg==",
"version": "5.4.11",
"resolved": "https://registry.npmjs.org/vite/-/vite-5.4.11.tgz",
"integrity": "sha512-c7jFQRklXua0mTzneGW9QVyxFjUgwcihC4bXEtujIo2ouWCe1Ajt/amn2PCxYnhYfd5k09JX3SB7OYWFKYqj8Q==",
"dev": true,
"license": "MIT",
"dependencies": {
"esbuild": "^0.21.3",
"postcss": "^8.4.40",
"rollup": "^4.13.0"
"postcss": "^8.4.43",
"rollup": "^4.20.0"
},
"bin": {
"vite": "bin/vite.js"
frontend/package.json
+5 -5
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.434.2",
"version": "1.435.2",
"scripts": {
"dev": "vite dev",
"build": "vite build",
@@ -21,10 +21,10 @@
"@hey-api/openapi-ts": "^0.43.0",
"@playwright/test": "^1.34.3",
"@rgossiaux/svelte-headlessui": "^2.0.0",
"@sveltejs/adapter-static": "^3.0.0",
"@sveltejs/kit": "^2.0.0",
"@sveltejs/adapter-static": "^3.0.6",
"@sveltejs/kit": "^2.9.0",
"@sveltejs/package": "^2.2.2",
"@sveltejs/vite-plugin-svelte": "^3.0.0",
"@sveltejs/vite-plugin-svelte": "^3.1.2",
"@tailwindcss/forms": "^0.5.3",
"@tailwindcss/typography": "^0.5.8",
"@types/d3": "^7.4.0",
@@ -64,7 +64,7 @@
"tailwindcss": "^3.4.1",
"tslib": "^2.6.1",
"typescript": "^5.1.3",
"vite": "^5",
"vite": "^5.4.11",
"vite-plugin-circular-dependency": "^0.2.1",
"vite-plugin-mkcert": "^1.17.5",
"yootils": "^0.3.1"

Some files were not shown because too many files have changed in this diff Show More