diff --git a/.github/change-versions-mac.sh b/.github/change-versions-mac.sh index 2257d348ae..50ec3b17ed 100755 --- a/.github/change-versions-mac.sh +++ b/.github/change-versions-mac.sh @@ -16,7 +16,7 @@ sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/ sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml -sed -i '' -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 +sed -i '' -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 # sed -i '' -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml sed -i '' -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile sed -i '' -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile diff --git a/.github/change-versions.sh b/.github/change-versions.sh index 77ecbc6985..699b87b0d7 100755 --- a/.github/change-versions.sh +++ b/.github/change-versions.sh @@ -17,7 +17,7 @@ sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/pac sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml -sed -i -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 +sed -i -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 # sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 42cc21264b..1f263e2696 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -76,7 +76,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,jemalloc,deno_core + features=embedding,parquet,openidconnect,jemalloc,deno_core,dind tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -138,7 +138,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} @@ -200,7 +200,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind PYTHON_IMAGE=python:3.12.2-slim-bookworm tags: | ${{ steps.meta-ee-public-py312.outputs.tags }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 29628db02c..94c2ef5cf0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,31 @@ # Changelog +## [1.435.2](https://github.com/windmill-labs/windmill/compare/v1.435.1...v1.435.2) (2024-12-05) + + +### Bug Fixes + +* job search toast on error ([#4851](https://github.com/windmill-labs/windmill/issues/4851)) ([a99e63f](https://github.com/windmill-labs/windmill/commit/a99e63f5435725c42e38b46933ff3caa345002b5)) + +## [1.435.1](https://github.com/windmill-labs/windmill/compare/v1.435.0...v1.435.1) (2024-12-05) + + +### Bug Fixes + +* improve critical alerts filters ([548cfcf](https://github.com/windmill-labs/windmill/commit/548cfcfbde23ac7ae129e10f6e92d57516e61f20)) + +## [1.435.0](https://github.com/windmill-labs/windmill/compare/v1.434.2...v1.435.0) (2024-12-05) + + +### Features + +* app custom paths ([#4828](https://github.com/windmill-labs/windmill/issues/4828)) ([1ec6c6f](https://github.com/windmill-labs/windmill/commit/1ec6c6f765904361e641d89495890bc87e8544aa)) + + +### Bug Fixes + +* pass USERPROFILE on windows ([5404ec9](https://github.com/windmill-labs/windmill/commit/5404ec9b48e8d7a0cb27b2319d54f04c94ec6fd0)) + ## [1.434.2](https://github.com/windmill-labs/windmill/compare/v1.434.1...v1.434.2) (2024-12-04) diff --git a/Dockerfile b/Dockerfile index cff245d3e4..ef2f7a721e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -175,9 +175,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools COPY --from=builder /frontend/build /static_frontend COPY --from=builder /windmill/target/release/windmill ${APP}/windmill -COPY --from=denoland/deno:2.0.4 --chmod=755 /usr/bin/deno /usr/bin/deno +COPY --from=denoland/deno:2.1.2 --chmod=755 /usr/bin/deno /usr/bin/deno -COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/backend/.sqlx/query-0a46f1f3047d15227f82ae24ad2113eb91d65b98927eaaba427cbde27dd79bfe.json b/backend/.sqlx/query-0a46f1f3047d15227f82ae24ad2113eb91d65b98927eaaba427cbde27dd79bfe.json new file mode 100644 index 0000000000..5bb7767b0e --- /dev/null +++ b/backend/.sqlx/query-0a46f1f3047d15227f82ae24ad2113eb91d65b98927eaaba427cbde27dd79bfe.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*)\n FROM alerts\n WHERE COALESCE(acknowledged, false) = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Bool" + ] + }, + "nullable": [ + null + ] + }, + "hash": "0a46f1f3047d15227f82ae24ad2113eb91d65b98927eaaba427cbde27dd79bfe" +} diff --git a/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json b/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json index 8fdca62eca..1b1c6b504b 100644 --- a/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json +++ b/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json @@ -44,7 +44,8 @@ "deploymentcallback", "singlescriptflow", "flowscript", - "flownode" + "flownode", + "appscript" ] } } diff --git a/backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json b/backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json new file mode 100644 index 0000000000..efe58b7a16 --- /dev/null +++ b/backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO app_script (app, hash, lock, code, code_sha256)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Bpchar", + "Text", + "Text", + "Bpchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b" +} diff --git a/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json b/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json new file mode 100644 index 0000000000..5c43791411 --- /dev/null +++ b/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT format('rawscript/%s', code_sha256) as \"path!: String\"\n FROM app_script WHERE id = $1 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!: String", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013" +} diff --git a/backend/.sqlx/query-1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20.json b/backend/.sqlx/query-1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20.json new file mode 100644 index 0000000000..51343088a6 --- /dev/null +++ b/backend/.sqlx/query-1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1ec97e1bf7c6edfa82b7e64585171ca897dcfa9e82618ce8f11afb08a39e3b20" +} diff --git a/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json b/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json index 20d76cbed4..2f80c0c818 100644 --- a/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json +++ b/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json @@ -34,7 +34,8 @@ "deploymentcallback", "singlescriptflow", "flowscript", - "flownode" + "flownode", + "appscript" ] } } diff --git a/backend/.sqlx/query-372eb162ace15a4f07162bb46706eaca91a3616c9bfbd78a408b7079ca9706d4.json b/backend/.sqlx/query-372eb162ace15a4f07162bb46706eaca91a3616c9bfbd78a408b7079ca9706d4.json new file mode 100644 index 0000000000..68fe6dba85 --- /dev/null +++ b/backend/.sqlx/query-372eb162ace15a4f07162bb46706eaca91a3616c9bfbd78a408b7079ca9706d4.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*)\n FROM alerts\n WHERE workspace_id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "372eb162ace15a4f07162bb46706eaca91a3616c9bfbd78a408b7079ca9706d4" +} diff --git a/backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json b/backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json new file mode 100644 index 0000000000..f4021d8985 --- /dev/null +++ b/backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bpchar", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d" +} diff --git a/backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json b/backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json new file mode 100644 index 0000000000..c9cd8219ee --- /dev/null +++ b/backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT policy as \"policy: sqlx::types::Json>\"\n FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "policy: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c" +} diff --git a/backend/.sqlx/query-65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a.json b/backend/.sqlx/query-65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a.json deleted file mode 100644 index e183d3b95c..0000000000 --- a/backend/.sqlx/query-65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE alerts\n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true\n ELSE acknowledged_workspace\n END\n WHERE id = $1", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int4", - "Text" - ] - }, - "nullable": [] - }, - "hash": "65da41c7ded54cdee8d33211561c068b72294cc99ff44ed0a13179df508ebc6a" -} diff --git a/backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json b/backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json similarity index 74% rename from backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json rename to backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json index 7e1f008dba..1521b9fd2f 100644 --- a/backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json +++ b/backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", + "query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at, worker_group FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "started_at", "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "worker_group", + "type_info": "Varchar" } ], "parameters": { @@ -43,8 +48,9 @@ true, true, false, + false, false ] }, - "hash": "a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7" + "hash": "6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973" } diff --git a/backend/.sqlx/query-75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2.json b/backend/.sqlx/query-6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5.json similarity index 63% rename from backend/.sqlx/query-75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2.json rename to backend/.sqlx/query-6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5.json index efaac63d39..0c82c9dd74 100644 --- a/backend/.sqlx/query-75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2.json +++ b/backend/.sqlx/query-6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "INSERT INTO app\n (workspace_id, path, summary, policy, versions, draft_only)\n VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id", + "query": "INSERT INTO app\n (workspace_id, path, summary, policy, versions, draft_only, custom_path)\n VALUES ($1, $2, $3, $4, '{}', $5, $6) RETURNING id", "describe": { "columns": [ { @@ -15,12 +15,13 @@ "Varchar", "Varchar", "Jsonb", - "Bool" + "Bool", + "Text" ] }, "nullable": [ false ] }, - "hash": "75e880f9d9fbda36c2314706923cef36e4667d930fb8ee1876dd9ce1c92396b2" + "hash": "6b53f7c4bb73177316d6134698f3979f51b53dcd4d8ec50d312c9e7fe31ad5f5" } diff --git a/backend/.sqlx/query-7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3.json b/backend/.sqlx/query-7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3.json deleted file mode 100644 index cf976de33f..0000000000 --- a/backend/.sqlx/query-7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "UPDATE alerts \n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $1::text IS NOT NULL THEN true\n ELSE acknowledged_workspace\n END\n WHERE ($1::text IS NOT NULL AND workspace_id = $1)\n OR ($1::text IS NULL)", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [] - }, - "hash": "7c32176755c6ea2b6ae531860d436caae3fa256fc0803749ec5107632669adb3" -} diff --git a/backend/.sqlx/query-80864158f61adaad8df934acc54ba523c9f17d106298d8781885134d28553d36.json b/backend/.sqlx/query-80864158f61adaad8df934acc54ba523c9f17d106298d8781885134d28553d36.json new file mode 100644 index 0000000000..5be45ac0fb --- /dev/null +++ b/backend/.sqlx/query-80864158f61adaad8df934acc54ba523c9f17d106298d8781885134d28553d36.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE alerts \n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $2 THEN\n CASE\n WHEN $1::text IS NOT NULL THEN true\n ELSE acknowledged_workspace\n END\n ELSE true\n END\n WHERE ($1::text IS NOT NULL AND workspace_id = $1)\n OR ($1::text IS NULL)", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "80864158f61adaad8df934acc54ba523c9f17d106298d8781885134d28553d36" +} diff --git a/backend/.sqlx/query-83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806.json b/backend/.sqlx/query-83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806.json new file mode 100644 index 0000000000..4e1242428a --- /dev/null +++ b/backend/.sqlx/query-83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bpchar", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [ + false + ] + }, + "hash": "83cc9e432aea1450f79e9fce04eca0e75f30faa8c39ad4754bd568ae6f210806" +} diff --git a/backend/.sqlx/query-951c053cf756f00e1da26b06edb3d0193a0ee707e6482a892db58915e0c8a27f.json b/backend/.sqlx/query-951c053cf756f00e1da26b06edb3d0193a0ee707e6482a892db58915e0c8a27f.json new file mode 100644 index 0000000000..7c51668501 --- /dev/null +++ b/backend/.sqlx/query-951c053cf756f00e1da26b06edb3d0193a0ee707e6482a892db58915e0c8a27f.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*)\n FROM alerts", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "951c053cf756f00e1da26b06edb3d0193a0ee707e6482a892db58915e0c8a27f" +} diff --git a/backend/.sqlx/query-b9ed42d4b795942251baafd016b4361e75257c0e5faf8795274ec86236a27413.json b/backend/.sqlx/query-b9ed42d4b795942251baafd016b4361e75257c0e5faf8795274ec86236a27413.json new file mode 100644 index 0000000000..40f0418eef --- /dev/null +++ b/backend/.sqlx/query-b9ed42d4b795942251baafd016b4361e75257c0e5faf8795274ec86236a27413.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*)\n FROM alerts\n WHERE workspace_id = $1 AND COALESCE(acknowledged_workspace, false) = $2", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Text", + "Bool" + ] + }, + "nullable": [ + null + ] + }, + "hash": "b9ed42d4b795942251baafd016b4361e75257c0e5faf8795274ec86236a27413" +} diff --git a/backend/.sqlx/query-cd4067e68b375461a495f402d05976da6c6e331d5748bf6f8d59f9f75c027fe8.json b/backend/.sqlx/query-cd4067e68b375461a495f402d05976da6c6e331d5748bf6f8d59f9f75c027fe8.json new file mode 100644 index 0000000000..1d1b577f7a --- /dev/null +++ b/backend/.sqlx/query-cd4067e68b375461a495f402d05976da6c6e331d5748bf6f8d59f9f75c027fe8.json @@ -0,0 +1,16 @@ +{ + "db_name": "PostgreSQL", + "query": "UPDATE alerts\n SET\n acknowledged = true,\n acknowledged_workspace = CASE\n WHEN $3 THEN\n CASE\n WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true\n ELSE acknowledged_workspace\n END\n ELSE true\n END\n WHERE id = $1", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int4", + "Text", + "Bool" + ] + }, + "nullable": [] + }, + "hash": "cd4067e68b375461a495f402d05976da6c6e331d5748bf6f8d59f9f75c027fe8" +} diff --git a/backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json b/backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json new file mode 100644 index 0000000000..450683521d --- /dev/null +++ b/backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT app_id, value FROM app_version WHERE id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "app_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "value", + "type_info": "Json" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false + ] + }, + "hash": "ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b" +} diff --git a/backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json b/backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json new file mode 100644 index 0000000000..0d76f9a721 --- /dev/null +++ b/backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app_version_lite (id, value) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602" +} diff --git a/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json b/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json index 4092b60b71..e48e3b041d 100644 --- a/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json +++ b/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json @@ -48,7 +48,8 @@ "deploymentcallback", "singlescriptflow", "flowscript", - "flownode" + "flownode", + "appscript" ] } } diff --git a/backend/.sqlx/query-fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df.json b/backend/.sqlx/query-fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df.json new file mode 100644 index 0000000000..a2362be620 --- /dev/null +++ b/backend/.sqlx/query-fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df.json @@ -0,0 +1,25 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text", + "Text", + "Text", + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "fb1399c1dc171ec6bb24fee3477a1d606d9725e20e9c2d76a0887fadfd87f8df" +} diff --git a/backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json b/backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json new file mode 100644 index 0000000000..49fe5cf6b8 --- /dev/null +++ b/backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "lock", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "code", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 54facad0c1..9599dbcf19 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1138,6 +1138,8 @@ dependencies = [ "cexpr", "clang-sys", "itertools 0.13.0", + "log", + "prettyplease 0.2.25", "proc-macro2", "quote", "regex", @@ -1254,6 +1256,50 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" +[[package]] +name = "bollard" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http 1.2.0", + "http-body-util", + "hyper 1.5.1", + "hyper-named-pipe", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 2.0.4", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.47.1-rc.27.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + [[package]] name = "borsh" version = "1.5.3" @@ -2440,9 +2486,9 @@ dependencies = [ [[package]] name = "deno_ast" -version = "0.42.2" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b9d03b1bbeeecdac54367f075d572131736d06c5be3bc49037855bc5ab1bbb" +checksum = "48d00b724e06d2081a141ec1155756a0b465d413d8e2a7515221f61d482eb2ee" dependencies = [ "base64 0.21.7", "deno_media_type", @@ -2481,18 +2527,18 @@ dependencies = [ [[package]] name = "deno_console" -version = "0.171.0" +version = "0.179.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144108c8bb93b1df2cda4583d9beb8cd4e18798d3e030af8b07d2c55c1e3259b" +checksum = "2e09f2bbb2d842329b602da25dbab5cd4a342f9a8adcb7c02509fc322f796e79" dependencies = [ "deno_core", ] [[package]] name = "deno_core" -version = "0.311.0" +version = "0.321.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e09bd55da542fa1fde753aff617c355b5d782e763ab2a19e4371a56d7844cac" +checksum = "cd2a54cda74cdc187d5fc2d23370a45cf09f912caf566dd1cd24a50157d809c7" dependencies = [ "anyhow", "bincode", @@ -2504,6 +2550,7 @@ dependencies = [ "deno_ops", "deno_unsync", "futures", + "indexmap 2.7.0", "libc", "memoffset", "parking_lot", @@ -2518,19 +2565,20 @@ dependencies = [ "tokio", "url", "v8", + "wasm_dep_analyzer", ] [[package]] name = "deno_core_icudata" -version = "0.0.73" +version = "0.74.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13951ea98c0a4c372f162d669193b4c9d991512de9f2381dd161027f34b26b1" +checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695" [[package]] name = "deno_fetch" -version = "0.195.0" +version = "0.203.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a91340a1d60cebe1392e4b6e1614709ab5fdcfc1f55842561e498d7ef9b2cb9" +checksum = "a18e66bd3bf786e24a8b8bdc97049fa82957b095a5fd1e142545c5a7cdd2272a" dependencies = [ "base64 0.21.7", "bytes", @@ -2540,6 +2588,7 @@ dependencies = [ "deno_tls", "dyn-clone", "error_reporter", + "hickory-resolver", "http 1.2.0", "http-body-util", "hyper 1.5.1", @@ -2550,6 +2599,7 @@ dependencies = [ "rustls-webpki 0.102.8", "serde", "serde_json", + "thiserror 1.0.69", "tokio", "tokio-rustls 0.26.0", "tokio-socks", @@ -2561,9 +2611,9 @@ dependencies = [ [[package]] name = "deno_media_type" -version = "0.1.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8978229b82552bf8457a0125aa20863f023619cfc21ebb007b1e571d68fd85b" +checksum = "eaa135b8a9febc9a51c16258e294e268a1276750780d69e46edb31cced2826e4" dependencies = [ "data-url", "serde", @@ -2585,31 +2635,33 @@ dependencies = [ [[package]] name = "deno_net" -version = "0.163.0" +version = "0.171.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b769fd37232a38bf15a3834c9da8cc99409b52d0cf4a936f3117744369f8063" +checksum = "f7b3a51f7b4d5d64d17a7bc6f7495498f20d809930979d21a059d75e850cdea6" dependencies = [ "deno_core", "deno_permissions", "deno_tls", + "hickory-proto", + "hickory-resolver", "pin-project", "rustls-tokio-stream", "serde", "socket2", + "thiserror 1.0.69", "tokio", - "trust-dns-proto", - "trust-dns-resolver", ] [[package]] name = "deno_ops" -version = "0.187.0" +version = "0.197.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e040fd4def8a67538fe38c9955fd970efc9f44284bd69d44f8992a456afd665d" +checksum = "37a8825d92301cf445727c43f17fee2a20fcdf4370004339965156ae7c56c97e" dependencies = [ "proc-macro-rules", "proc-macro2", "quote", + "stringcase", "strum 0.25.0", "strum_macros 0.25.3", "syn 2.0.90", @@ -2618,9 +2670,9 @@ dependencies = [ [[package]] name = "deno_path_util" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4889646c1ce8437a6fde3acb057fd7e2d039e62c61f5063fc125ed1ede114dc6" +checksum = "ff25f6e08e7a0214bbacdd6f7195c7f1ebcd850c87a624e4ff06326b68b42d99" dependencies = [ "percent-encoding", "thiserror 1.0.69", @@ -2629,9 +2681,9 @@ dependencies = [ [[package]] name = "deno_permissions" -version = "0.31.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1328c2d1d26cd066ba9d7d5fb3451081219e373342423f78d7a1b73bfac9849" +checksum = "14e822f98185ab3ddf06104b2407681e0008af52361af32f1cd171b7eda5aa59" dependencies = [ "deno_core", "deno_path_util", @@ -2642,6 +2694,7 @@ dependencies = [ "once_cell", "percent-encoding", "serde", + "thiserror 1.0.69", "which 4.4.2", "winapi", ] @@ -2668,9 +2721,9 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.158.0" +version = "0.166.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a1abc6bd8af41aa2496ceceef94f4277092c781a9495c437327a17659553a2" +checksum = "688175eed35e7b3053ec114227894ef24786855405d8844058a48bffa997d85a" dependencies = [ "deno_core", "deno_native_certs", @@ -2679,6 +2732,7 @@ dependencies = [ "rustls-tokio-stream", "rustls-webpki 0.102.8", "serde", + "thiserror 1.0.69", "tokio", "webpki-roots 0.26.7", ] @@ -2696,19 +2750,20 @@ dependencies = [ [[package]] name = "deno_url" -version = "0.171.0" +version = "0.179.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68eb6834d66ff13c19633b0e1b621292461b744858130b0a1c51f34a57ba1a03" +checksum = "ad9a108794e505f2b07665e19ff336c1bcba6adcf7182c90c1d3a6c741d7fcd0" dependencies = [ "deno_core", + "thiserror 1.0.69", "urlpattern", ] [[package]] name = "deno_web" -version = "0.202.0" +version = "0.210.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0862246372f5b559b788aa07ca1b9385909d59630251156a54c4d2c5342d1eb" +checksum = "7679087bcc41f7ae3385f8c12d43bc81cfc54cb9b1ef73983d20f5e39fa4e0da" dependencies = [ "async-trait", "base64-simd 0.8.0", @@ -2719,15 +2774,16 @@ dependencies = [ "flate2", "futures", "serde", + "thiserror 1.0.69", "tokio", "uuid 1.11.0", ] [[package]] name = "deno_webidl" -version = "0.171.0" +version = "0.179.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e969b61b740479379eaf303a6900d11f162f9de610640398f89f0aa58abb7c4" +checksum = "5b55d845e3d64f8de7eff67aaa4b6fe1b23bbc2efe967c984f8c64c8dd85fad4" dependencies = [ "deno_core", ] @@ -3941,6 +3997,53 @@ dependencies = [ "ureq", ] +[[package]] +name = "hickory-proto" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.4.0", + "ipnet", + "once_cell", + "rand 0.8.5", + "serde", + "thiserror 1.0.69", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28757f23aa75c98f254cf0405e6d8c25b831b32921b050a66692427679b1f243" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto", + "ipconfig", + "lru-cache", + "once_cell", + "parking_lot", + "rand 0.8.5", + "resolv-conf", + "serde", + "smallvec", + "thiserror 1.0.69", + "tokio", + "tracing", +] + [[package]] name = "hkdf" version = "0.12.4" @@ -4139,6 +4242,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper 1.5.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -4173,6 +4291,19 @@ dependencies = [ "tower-service", ] +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper 1.5.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -4222,6 +4353,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.5.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.61" @@ -4809,7 +4955,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "674883a98273598ac3aad4301724c56734bea90574c5033af067e8f9fb5eb399" dependencies = [ - "prost", + "prost 0.12.6", "prost-types", ] @@ -5109,13 +5255,13 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -5637,6 +5783,91 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f3cebff57f7dbd1255b44d8bddc2cebeb0ea677dbaa2e25a3070a91b318f660" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror 1.0.69", +] + +[[package]] +name = "opentelemetry-appender-tracing" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5feffc321035ad94088a7e5333abb4d84a8726e54a802e736ce9dd7237e85b" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +dependencies = [ + "async-trait", + "futures-core", + "http 1.2.0", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost 0.13.3", + "thiserror 1.0.69", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost 0.13.3", + "tonic", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" + +[[package]] +name = "opentelemetry_sdk" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b742c1cae4693792cc564e58d75a2a0ba29421a34a85b50da92efa89ecb2bc" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "percent-encoding", + "rand 0.8.5", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -6264,7 +6495,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.6", +] + +[[package]] +name = "prost" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b0487d90e047de87f984913713b85c601c05609aad5b0df4b4573fbf69aa13f" +dependencies = [ + "bytes", + "prost-derive 0.13.3", ] [[package]] @@ -6280,13 +6521,26 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "prost-derive" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9552f850d5f0964a4e4d0bf306459ac29323ddfbae05e35a7c0d35cb0803cc5" +dependencies = [ + "anyhow", + "itertools 0.13.0", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "prost-types" version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ - "prost", + "prost 0.12.6", ] [[package]] @@ -7564,6 +7818,17 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "serde_spanned" version = "0.6.8" @@ -7599,9 +7864,9 @@ dependencies = [ [[package]] name = "serde_v8" -version = "0.220.0" +version = "0.230.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7a65d91d79acc82aa229aeb084f4a39bda269069bc1520df40f679495388e4" +checksum = "b5a783242d2af51d6955cc04bf2b64adb643ab588b61e9573c908a69dabf8c2f" dependencies = [ "num-bigint", "serde", @@ -8248,6 +8513,12 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "stringcase" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04028eeb851ed08af6aba5caa29f2d59a13ed168cee4d6bd753aeefcf1d636b0" + [[package]] name = "stringprep" version = "0.1.5" @@ -9237,29 +9508,28 @@ dependencies = [ [[package]] name = "tokio" -version = "1.36.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", "tracing", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", @@ -9448,6 +9718,39 @@ dependencies = [ "winnow 0.6.20", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2 0.4.7", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.5.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.3", + "rustls-native-certs 0.8.1", + "rustls-pemfile 2.2.0", + "socket2", + "tokio", + "tokio-rustls 0.26.0", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "toolchain_find" version = "0.4.0" @@ -9469,11 +9772,16 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand 0.8.5", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -9651,6 +9959,24 @@ dependencies = [ "url", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log 0.2.0", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -9702,54 +10028,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "trust-dns-proto" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3119112651c157f4488931a01e586aa459736e9d6046d3bd9105ffb69352d374" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna 0.4.0", - "ipnet", - "once_cell", - "rand 0.8.5", - "serde", - "smallvec", - "thiserror 1.0.69", - "tinyvec", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "trust-dns-resolver" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a3e6c3aff1718b3c73e395d1f35202ba2ffa847c6a62eea0db8fb4cfe30be6" -dependencies = [ - "cfg-if", - "futures-util", - "ipconfig", - "lru-cache", - "once_cell", - "parking_lot", - "rand 0.8.5", - "resolv-conf", - "serde", - "smallvec", - "thiserror 1.0.69", - "tokio", - "tracing", - "trust-dns-proto", -] - [[package]] name = "try-lock" version = "0.2.5" @@ -10142,11 +10420,11 @@ dependencies = [ [[package]] name = "v8" -version = "0.106.0" +version = "130.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a381badc47c6f15acb5fe0b5b40234162349ed9d4e4fd7c83a7f5547c0fc69c5" +checksum = "2ee0be58935708fa4d7efb970c6cf9f2d9511d24ee24246481a65b6ee167348d" dependencies = [ - "bindgen 0.69.5", + "bindgen 0.70.1", "bitflags 2.6.0", "fslock", "gzip-header", @@ -10328,6 +10606,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm_dep_analyzer" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f270206a91783fd90625c8bb0d8fbd459d0b1d1bf209b656f713f01ae7c04b8" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "wav" version = "1.0.1" @@ -10446,7 +10733,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "axum", @@ -10487,7 +10774,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "argon2", @@ -10573,7 +10860,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.434.2" +version = "1.435.2" dependencies = [ "base64 0.22.1", "chrono", @@ -10591,7 +10878,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.434.2" +version = "1.435.2" dependencies = [ "chrono", "serde", @@ -10604,7 +10891,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "serde", @@ -10618,7 +10905,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "async-stream", @@ -10644,6 +10931,12 @@ dependencies = [ "magic-crypt", "mail-send", "object_store", + "opentelemetry", + "opentelemetry-appender-tracing", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", + "pin-project-lite", "prometheus", "quick_cache", "rand 0.8.5", @@ -10657,10 +10950,12 @@ dependencies = [ "thiserror 2.0.4", "tikv-jemalloc-ctl", "tokio", + "tonic", "tracing", "tracing-appender", "tracing-flame", "tracing-loki", + "tracing-opentelemetry", "tracing-subscriber", "uuid 1.11.0", "windmill-macros", @@ -10668,7 +10963,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.434.2" +version = "1.435.2" dependencies = [ "regex", "serde", @@ -10682,7 +10977,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "bytes", @@ -10705,7 +11000,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.434.2" +version = "1.435.2" dependencies = [ "itertools 0.13.0", "lazy_static", @@ -10717,7 +11012,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.434.2" +version = "1.435.2" dependencies = [ "convert_case 0.6.0", "serde", @@ -10726,7 +11021,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "lazy_static", @@ -10738,7 +11033,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "gosyn", @@ -10750,7 +11045,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "lazy_static", @@ -10762,7 +11057,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10773,7 +11068,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10784,7 +11079,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "async-recursion", @@ -10802,7 +11097,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10819,7 +11114,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "lazy_static", @@ -10831,7 +11126,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "lazy_static", @@ -10849,7 +11144,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10870,7 +11165,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "serde_json", @@ -10880,7 +11175,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "async-recursion", @@ -10896,6 +11191,7 @@ dependencies = [ "hmac", "itertools 0.13.0", "lazy_static", + "opentelemetry", "prometheus", "regex", "reqwest 0.12.9", @@ -10914,7 +11210,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.434.2" +version = "1.435.2" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10924,13 +11220,14 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.434.2" +version = "1.435.2" dependencies = [ "anyhow", "async-recursion", "backon", "base64 0.22.1", "bit-vec", + "bollard", "bytes", "chrono", "const_format", @@ -10940,6 +11237,7 @@ dependencies = [ "deno_core", "deno_fetch", "deno_net", + "deno_permissions", "deno_tls", "deno_url", "deno_web", @@ -10960,6 +11258,7 @@ dependencies = [ "object_store", "once_cell", "openidconnect", + "opentelemetry", "pem 3.0.4", "postgres-native-tls", "prometheus", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index be954faac2..88b82d896b 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.434.2" +version = "1.435.2" authors.workspace = true edition.workspace = true @@ -29,7 +29,7 @@ members = [ ] [workspace.package] -version = "1.434.2" +version = "1.435.2" authors = ["Ruben Fiszel "] edition = "2021" @@ -63,6 +63,8 @@ tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"] sqlx = ["windmill-worker/sqlx"] deno_core = ["windmill-worker/deno_core", "dep:deno_core"] kafka = ["windmill-api/kafka"] +otel = ["windmill-common/otel", "windmill-worker/otel"] +dind = ["windmill-worker/dind"] [dependencies] anyhow.workspace = true @@ -95,7 +97,6 @@ deno_core = { workspace = true, optional = true } object_store = { workspace = true, optional = true } quote.workspace = true - [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } tikv-jemalloc-sys = { optional = true, workspace = true } @@ -137,7 +138,7 @@ windmill-api-client = { path = "./windmill-api-client" } axum = { version = "^0.7", features = ["multipart"] } headers = "^0" hyper = { version = "^1", features = ["full"] } -tokio = { version = "^1", features = ["full", "tracing"] } +tokio = { version = "^1.42.0", features = ["full", "tracing"] } tower = "^0" tower-http = { version = "^0.6", features = ["trace", "cors"] } tower-cookies = "^0.10" @@ -182,15 +183,16 @@ itertools = "^0" regex = "^1" semver = "^1" -deno_fetch = "0.195.0" -deno_tls = "0.158.0" -deno_console = "0.171.0" -deno_url = "0.171.0" -deno_webidl = "0.171.0" -deno_web = "0.202.0" -deno_net = "0.163.0" -deno_core = "0.311.0" -deno_ast = { version = "=0.42.2", features = ["transpiling"] } +deno_fetch = "0.203.0" +deno_tls = "0.166.0" +deno_console = "0.179.0" +deno_url = "0.179.0" +deno_webidl = "0.179.0" +deno_web = "0.210.0" +deno_net = "0.171.0" +deno_core = "0.321.0" +deno_ast = { version = "=0.43.3", features = ["transpiling"] } +deno_permissions = "0.39.0" swc_common = "=0.37.5" swc_ecma_parser = "=0.149.1" @@ -279,11 +281,23 @@ tar = "^0" http = "^1" async-stream = "^0" +opentelemetry = "0.27.0" +tracing-opentelemetry = "0.28.0" +opentelemetry_sdk = { version = "*", features = ["rt-tokio"] } +opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] } +opentelemetry-appender-tracing = "0.27.0" +opentelemetry-semantic-conventions = { version = "*", features = ["semconv_experimental"] } + +bollard = "0.18.1" + +tonic = { version = "^0", features = ["tls-native-roots"] } + tikv-jemallocator = { version = "0.5" } tikv-jemalloc-sys = { version = "^0.5" } tikv-jemalloc-ctl = { version = "^0.5" } triomphe = "^0" +pin-project-lite = "^0" tantivy = "0.22.0" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 496432b7e0..32341c9e08 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -aefbc1e2188fea312996fcfc30a29d8fb5315316 \ No newline at end of file +a07bc62582c809457f1c945d6cda145770b94d04 \ No newline at end of file diff --git a/backend/migrations/20241202095902_app_script.down.sql b/backend/migrations/20241202095902_app_script.down.sql new file mode 100644 index 0000000000..c493d82176 --- /dev/null +++ b/backend/migrations/20241202095902_app_script.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP TABLE IF EXISTS app_version_lite; +DROP TABLE IF EXISTS app_script; diff --git a/backend/migrations/20241202095902_app_script.up.sql b/backend/migrations/20241202095902_app_script.up.sql new file mode 100644 index 0000000000..5c3e6a2e0b --- /dev/null +++ b/backend/migrations/20241202095902_app_script.up.sql @@ -0,0 +1,26 @@ +-- Add up migration script here +ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'appscript'; + +-- Same as `app_version` but with a "lite" value (w/ `inlineScript.{code,lock}`). +CREATE TABLE app_version_lite ( + id BIGSERIAL PRIMARY KEY, + value JSONB, + FOREIGN KEY (id) REFERENCES app_version (id) ON DELETE CASCADE +); + +GRANT ALL ON app_version_lite TO windmill_user; +GRANT ALL ON app_version_lite TO windmill_admin; + +-- App `inlineScript`. +CREATE TABLE app_script ( + id BIGSERIAL PRIMARY KEY, + app BIGSERIAL NOT NULL, + hash CHAR(64) NOT NULL UNIQUE, -- sha256 of `app`, `lock`, `code`. + lock TEXT, + code TEXT NOT NULL, + code_sha256 CHAR(64) NOT NULL, -- used to retrieve the policy. + FOREIGN KEY (app) REFERENCES app (id) ON DELETE CASCADE +); + +GRANT ALL ON app_script TO windmill_user; +GRANT ALL ON app_script TO windmill_admin; diff --git a/backend/migrations/20241202134622_app_custom_path.down.sql b/backend/migrations/20241202134622_app_custom_path.down.sql new file mode 100644 index 0000000000..1dadbefff1 --- /dev/null +++ b/backend/migrations/20241202134622_app_custom_path.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE app DROP COLUMN custom_path; diff --git a/backend/migrations/20241202134622_app_custom_path.up.sql b/backend/migrations/20241202134622_app_custom_path.up.sql new file mode 100644 index 0000000000..832efd0cb2 --- /dev/null +++ b/backend/migrations/20241202134622_app_custom_path.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE app ADD COLUMN custom_path TEXT CHECK (custom_path ~ '^[\w-]+(\/[\w-]+)*$'); diff --git a/backend/migrations/20241204154025_grant_all_concurrency_key.down.sql b/backend/migrations/20241204154025_grant_all_concurrency_key.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20241204154025_grant_all_concurrency_key.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20241204154025_grant_all_concurrency_key.up.sql b/backend/migrations/20241204154025_grant_all_concurrency_key.up.sql new file mode 100644 index 0000000000..da9b9704d7 --- /dev/null +++ b/backend/migrations/20241204154025_grant_all_concurrency_key.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +GRANT ALL ON concurrency_key TO windmill_admin; +GRANT ALL ON concurrency_key TO windmill_user; \ No newline at end of file diff --git a/backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql b/backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql new file mode 100644 index 0000000000..7a5d08f3f8 --- /dev/null +++ b/backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE flow_node DROP COLUMN hash_v2; diff --git a/backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql b/backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql new file mode 100644 index 0000000000..799f33d341 --- /dev/null +++ b/backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql @@ -0,0 +1,4 @@ +-- Add up migration script here +CREATE SEQUENCE IF NOT EXISTS flow_node_hash_seq; +ALTER TABLE flow_node ALTER COLUMN hash DROP NOT NULL; +ALTER TABLE flow_node ADD COLUMN hash_v2 CHAR(64) NOT NULL UNIQUE DEFAULT to_hex(nextval('flow_node_hash_seq')); diff --git a/backend/migrations/20241206075559_flow_node_unique_2.down.sql b/backend/migrations/20241206075559_flow_node_unique_2.down.sql new file mode 100644 index 0000000000..b6e6db03e6 --- /dev/null +++ b/backend/migrations/20241206075559_flow_node_unique_2.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_unique_2; +ALTER TABLE flow_node ADD CONSTRAINT flow_node_hash_v2_key UNIQUE (hash_v2); diff --git a/backend/migrations/20241206075559_flow_node_unique_2.up.sql b/backend/migrations/20241206075559_flow_node_unique_2.up.sql new file mode 100644 index 0000000000..4e8383bd89 --- /dev/null +++ b/backend/migrations/20241206075559_flow_node_unique_2.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +ALTER TABLE flow_node ADD CONSTRAINT flow_node_unique_2 UNIQUE (path, workspace_id, hash_v2); +ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_hash_v2_key; diff --git a/backend/src/main.rs b/backend/src/main.rs index d403ad212a..e76dfa79f6 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -8,7 +8,7 @@ use anyhow::Context; use monitor::{ - reload_delete_logs_periodically_setting, reload_indexer_config, + load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store, }; @@ -37,7 +37,7 @@ use windmill_common::{ EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, - OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, @@ -221,11 +221,66 @@ async fn windmill_main() -> anyhow::Result<()> { let hostname = hostname(); - #[cfg(not(feature = "flamegraph"))] - let _guard = windmill_common::tracing_init::initialize_tracing(&hostname); + let mut enable_standalone_indexer: bool = false; + + let mode = std::env::var("MODE") + .map(|x| x.to_lowercase()) + .map(|x| { + if &x == "server" { + println!("Binary is in 'server' mode"); + Mode::Server + } else if &x == "worker" { + tracing::info!("Binary is in 'worker' mode"); + #[cfg(windows)] + { + println!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL"); + } + Mode::Worker + } else if &x == "agent" { + println!("Binary is in 'agent' mode"); + if std::env::var("BASE_INTERNAL_URL").is_err() { + panic!("BASE_INTERNAL_URL is required in agent mode") + } + if std::env::var("JOB_TOKEN").is_err() { + println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table") + } + + #[cfg(not(feature = "enterprise"))] + { + panic!("Agent mode is only available in the EE, ignoring..."); + } + #[cfg(feature = "enterprise")] + Mode::Agent + } else if &x == "indexer" { + tracing::info!("Binary is in 'indexer' mode"); + #[cfg(not(feature = "tantivy"))] + { + eprintln!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features."); + panic!("Indexer mode requires compiling with the tantivy feature flag."); + } + #[cfg(feature = "tantivy")] + Mode::Indexer + } else if &x == "standalone+search"{ + enable_standalone_indexer = true; + println!("Binary is in 'standalone' mode with search enabled"); + Mode::Standalone + } + else { + if &x != "standalone" { + eprintln!("mode not recognized, defaulting to standalone: {x}"); + } else { + println!("Binary is in 'standalone' mode"); + } + Mode::Standalone + } + }) + .unwrap_or_else(|_| { + tracing::info!("Mode not specified, defaulting to standalone"); + Mode::Standalone + }); #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] - tracing::info!("jemalloc enabled"); + println!("jemalloc enabled"); #[cfg(feature = "flamegraph")] let _guard = windmill_common::tracing_init::setup_flamegraph(); @@ -236,13 +291,13 @@ async fn windmill_main() -> anyhow::Result<()> { "cache" => { #[cfg(feature = "embedding")] { - tracing::info!("Caching embedding model..."); + println!("Caching embedding model..."); windmill_api::embeddings::ModelInstance::load_model_files().await?; - tracing::info!("Cached embedding model"); + println!("Cached embedding model"); } #[cfg(not(feature = "embedding"))] { - tracing::warn!("Embeddings are not enabled, ignoring..."); + println!("Embeddings are not enabled, ignoring..."); } cache_hub_scripts(std::env::args().nth(2)).await?; @@ -256,64 +311,6 @@ async fn windmill_main() -> anyhow::Result<()> { _ => {} } - let mut enable_standalone_indexer: bool = false; - - let mode = std::env::var("MODE") - .map(|x| x.to_lowercase()) - .map(|x| { - if &x == "server" { - tracing::info!("Binary is in 'server' mode"); - Mode::Server - } else if &x == "worker" { - tracing::info!("Binary is in 'worker' mode"); - #[cfg(windows)] - { - tracing::warn!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL"); - } - Mode::Worker - } else if &x == "agent" { - tracing::info!("Binary is in 'agent' mode"); - if std::env::var("BASE_INTERNAL_URL").is_err() { - panic!("BASE_INTERNAL_URL is required in agent mode") - } - if std::env::var("JOB_TOKEN").is_err() { - tracing::warn!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table") - } - - #[cfg(not(feature = "enterprise"))] - { - panic!("Agent mode is only available in the EE, ignoring..."); - } - #[cfg(feature = "enterprise")] - Mode::Agent - } else if &x == "indexer" { - tracing::info!("Binary is in 'indexer' mode"); - #[cfg(not(feature = "tantivy"))] - { - tracing::error!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features."); - panic!("Indexer mode requires compiling with the tantivy feature flag."); - } - #[cfg(feature = "tantivy")] - Mode::Indexer - } else if &x == "standalone+search"{ - enable_standalone_indexer = true; - tracing::info!("Binary is in 'standalone' mode with search enabled"); - Mode::Standalone - } - else { - if &x != "standalone" { - tracing::error!("mode not recognized, defaulting to standalone: {x}"); - } else { - tracing::info!("Binary is in 'standalone' mode"); - } - Mode::Standalone - } - }) - .unwrap_or_else(|_| { - tracing::info!("Mode not specified, defaulting to standalone"); - Mode::Standalone - }); - #[allow(unused_mut)] let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer { 0 @@ -325,7 +322,7 @@ async fn windmill_main() -> anyhow::Result<()> { }; if num_workers > 1 { - tracing::warn!( + println!( "We STRONGLY recommend using at most 1 worker per container, use at your own risks" ); } @@ -347,10 +344,26 @@ async fn windmill_main() -> anyhow::Result<()> { IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)) }; - tracing::info!("Connecting to database..."); + println!("Connecting to database..."); let db = windmill_common::connect_db(server_mode, indexer_mode).await?; + + load_otel(&db).await; + tracing::info!("Database connected"); + let environment = load_base_url(&db) + .await + .unwrap_or_else(|_| "local".to_string()) + .trim_start_matches("https://") + .trim_start_matches("http://") + .split(".") + .next() + .unwrap_or_else(|| "local") + .to_string(); + + #[cfg(not(feature = "flamegraph"))] + let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment); + let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await; tracing::info!( @@ -776,6 +789,15 @@ Windmill Community Edition {GIT_VERSION} tracing::error!(error = %e, "Could not reload debug metrics setting"); } }, + OTEL_SETTING => { + tracing::info!("OTEL setting changed, restarting"); + // we wait a bit randomly to avoid having all servers and workers shutdown at same time + let rd_delay = rand::thread_rng().gen_range(0..4); + tokio::time::sleep(Duration::from_secs(rd_delay)).await; + if let Err(e) = tx.send(()) { + tracing::error!(error = %e, "Could not send killpill"); + } + }, REQUEST_SIZE_LIMIT_SETTING => { if server_mode { tracing::info!("Request limit size change detected, killing server expecting to be restarted"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 733a841283..c11d60621e 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -12,7 +12,7 @@ use std::{ use chrono::{NaiveDateTime, Utc}; use futures::{stream::FuturesUnordered, StreamExt}; -use serde::de::DeserializeOwned; +use serde::{de::DeserializeOwned, Deserializer}; use sqlx::{Pool, Postgres}; use tokio::{ join, @@ -40,7 +40,7 @@ use windmill_common::{ EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, - PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, @@ -58,7 +58,8 @@ use windmill_common::{ }, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, - MONITOR_LOGS_ON_OBJECT_STORE, SERVICE_LOG_RETENTION_SECS, + MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, + SERVICE_LOG_RETENTION_SECS, }; use windmill_queue::cancel_job; use windmill_worker::{ @@ -85,12 +86,12 @@ lazy_static::lazy_static! { static ref ZOMBIE_JOB_TIMEOUT: String = std::env::var("ZOMBIE_JOB_TIMEOUT") .ok() .and_then(|x| x.parse::().ok()) - .unwrap_or_else(|| "30".to_string()); + .unwrap_or_else(|| "60".to_string()); static ref FLOW_ZOMBIE_TRANSITION_TIMEOUT: String = std::env::var("FLOW_ZOMBIE_TRANSITION_TIMEOUT") .ok() .and_then(|x| x.parse::().ok()) - .unwrap_or_else(|| "30".to_string()); + .unwrap_or_else(|| "60".to_string()); pub static ref RESTART_ZOMBIE_JOBS: bool = std::env::var("RESTART_ZOMBIE_JOBS") @@ -199,6 +200,65 @@ pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> { Ok(()) } +fn empty_string_as_none<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let option = as serde::Deserialize>::deserialize(deserializer)?; + Ok(option.filter(|s| !s.is_empty())) +} + +#[derive(serde::Deserialize)] +struct OtelSetting { + metrics_enabled: Option, + logs_enabled: Option, + tracing_enabled: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_endpoint: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_headers: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_protocol: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_compression: Option, +} + +pub async fn load_otel(db: &DB) { + let otel = load_value_from_global_settings(db, OTEL_SETTING).await; + if let Ok(v) = otel { + if let Some(v) = v { + let deser = serde_json::from_value::(v); + if let Ok(o) = deser { + let metrics_enabled = o.metrics_enabled.unwrap_or(false); + let logs_enabled = o.logs_enabled.unwrap_or(false); + let tracing_enabled = o.tracing_enabled.unwrap_or(false); + + OTEL_METRICS_ENABLED.store(metrics_enabled, Ordering::Relaxed); + OTEL_LOGS_ENABLED.store(logs_enabled, Ordering::Relaxed); + OTEL_TRACING_ENABLED.store(tracing_enabled, Ordering::Relaxed); + if let Some(endpoint) = o.otel_exporter_otlp_endpoint.as_ref() { + std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint); + } + if let Some(headers) = o.otel_exporter_otlp_headers.as_ref() { + std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", headers); + } + if let Some(protocol) = o.otel_exporter_otlp_protocol { + std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", protocol); + } + if let Some(compression) = o.otel_exporter_otlp_compression { + std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", compression); + } + println!("OTEL settings loaded: tracing ({tracing_enabled}), logs ({logs_enabled}), metrics ({metrics_enabled}), endpoint ({:?}), headers defined: ({})", + o.otel_exporter_otlp_endpoint, o.otel_exporter_otlp_headers.is_some()); + } else { + tracing::error!("Error deserializing otel settings"); + } + } + } else { + tracing::error!("Error loading otel settings: {}", otel.unwrap_err()); + } +} + pub async fn load_tag_per_workspace_enabled(db: &DB) -> error::Result<()> { let metrics_enabled = load_value_from_global_settings(db, DEFAULT_TAGS_PER_WORKSPACE_SETTING).await; @@ -1337,7 +1397,7 @@ pub async fn reload_worker_config( } } -pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { +pub async fn load_base_url(db: &DB) -> error::Result { let q_base_url = load_value_from_global_settings(db, BASE_URL_SETTING).await?; let std_base_url = std::env::var("BASE_URL") @@ -1361,6 +1421,14 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { std_base_url }; + { + let mut l = BASE_URL.write().await; + *l = base_url.clone(); + } + Ok(base_url) +} + +pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?; let oauths = if let Some(q) = q_oauth { @@ -1374,6 +1442,7 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { None }; + let base_url = load_base_url(db).await?; let is_secure = base_url.starts_with("https://"); { @@ -1383,11 +1452,6 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { .unwrap(); } - { - let mut l = BASE_URL.write().await; - *l = base_url - } - { let mut l = IS_SECURE.write().await; *l = is_secure; diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index a229450651..571b71cd8f 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -77,7 +77,11 @@ async fn initialize_tracing() { static ONCE: Once = Once::new(); ONCE.call_once(|| { - let _ = windmill_common::tracing_init::initialize_tracing("test"); + let _ = windmill_common::tracing_init::initialize_tracing( + "test", + &windmill_common::utils::Mode::Standalone, + "test", + ); }); } diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 0e7502d6cd..c8893f824c 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -8709,6 +8709,9 @@ paths: - identity - deploymentcallback - singlescriptflow + - flowscript + - flownode + - appscript schedule_path: type: string permissioned_as: @@ -9267,6 +9270,9 @@ paths: - identity - deploymentcallback - singlescriptflow + - flowscript + - flownode + - appscript schedule_path: type: string permissioned_as: diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 4a9492418b..0bce856613 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.434.2 + version: 1.435.2 title: Windmill API contact: @@ -872,9 +872,20 @@ paths: content: application/json: schema: - type: array - items: - $ref: '#/components/schemas/CriticalAlert' + type: object + properties: + alerts: + type: array + items: + $ref: '#/components/schemas/CriticalAlert' + total_rows: + type: integer + description: Total number of rows matching the query. + example: 100 + total_pages: + type: integer + description: Total number of pages based on the page size. + example: 10 /settings/critical_alerts/{id}/acknowledge: post: @@ -2757,9 +2768,20 @@ paths: content: application/json: schema: - type: array - items: - $ref: '#/components/schemas/CriticalAlert' + type: object + properties: + alerts: + type: array + items: + $ref: '#/components/schemas/CriticalAlert' + total_rows: + type: integer + description: Total number of rows matching the query. + example: 100 + total_pages: + type: integer + description: Total number of pages based on the page size. + example: 10 /w/{workspace}/workspaces/critical_alerts/{id}/acknowledge: post: @@ -3733,6 +3755,27 @@ paths: required: - app + /apps_u/public_app_by_custom_path/{custom_path}: + get: + summary: get public app by custom path + operationId: getPublicAppByCustomPath + tags: + - app + parameters: + - $ref: "#/components/parameters/CustomPath" + responses: + "200": + description: app details + content: + application/json: + schema: + allOf: + - $ref: "#/components/schemas/AppWithLastVersion" + - type: object + properties: + workspace_id: + type: string + /scripts/hub/get/{path}: get: summary: get hub script content by path @@ -5371,6 +5414,8 @@ paths: type: boolean deployment_message: type: string + custom_path: + type: string required: - path - value @@ -5422,6 +5467,23 @@ paths: schema: $ref: "#/components/schemas/AppWithLastVersion" + /w/{workspace}/apps/get/lite/{path}: + get: + summary: get app lite by path + operationId: getAppLiteByPath + tags: + - app + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: app lite details + content: + application/json: + schema: + $ref: "#/components/schemas/AppWithLastVersion" + /w/{workspace}/apps/get/draft/{path}: get: summary: get app by path with draft @@ -5696,6 +5758,8 @@ paths: $ref: "#/components/schemas/Policy" deployment_message: type: string + custom_path: + type: string responses: "200": description: app updated @@ -5704,6 +5768,23 @@ paths: schema: type: string + /w/{workspace}/apps/custom_path_exists/{custom_path}: + get: + summary: check if custom path exists + operationId: customPathExists + tags: + - app + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/CustomPath" + responses: + "200": + description: custom path exists + content: + application/json: + schema: + type: boolean + /w/{workspace}/apps_u/execute_component/{path}: post: summary: executeComponent @@ -5727,6 +5808,8 @@ paths: #flow: flow/ path: type: string + version: + type: integer args: {} raw_code: type: object @@ -5744,6 +5827,8 @@ paths: required: - content - language + id: + type: integer force_viewer_static_fields: type: object force_viewer_one_of_fields: @@ -10222,6 +10307,12 @@ components: required: true schema: type: string + CustomPath: + name: custom_path + in: path + required: true + schema: + type: string PathId: name: id in: path @@ -10892,6 +10983,8 @@ components: "deploymentcallback", "singlescriptflow", "flowscript", + "flownode", + "appscript", ] schedule_path: type: string @@ -11012,6 +11105,8 @@ components: "deploymentcallback", "singlescriptflow", "flowscript", + "flownode", + "appscript", ] schedule_path: type: string @@ -12860,6 +12955,8 @@ components: draft_only: type: boolean draft: {} + custom_path: + type: string AppHistory: type: object diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index 5e236baeb6..af167d9a4a 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; /* * Author: Ruben Fiszel @@ -31,6 +31,7 @@ use axum::{ routing::{delete, get, post}, Router, }; +use futures::future::{FutureExt, TryFutureExt}; use hyper::StatusCode; #[cfg(feature = "parquet")] use itertools::Itertools; @@ -50,16 +51,17 @@ use windmill_audit::ActionKind; #[cfg(feature = "parquet")] use windmill_common::s3_helpers::build_object_store_client; use windmill_common::{ - apps::ListAppQuery, + apps::{AppScriptId, ListAppQuery}, + cache::{self, future::FutureCachedExt}, db::UserDB, error::{to_anyhow, Error, JsonResult, Result}, jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode}, users::username_to_permissioned_as, utils::{ - http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, Pagination, StripPath, + http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath }, variables::{build_crypt, build_crypt_with_key_suffix}, - worker::to_raw_value, + worker::{to_raw_value, CLOUD_HOSTED}, HUB_BASE_URL, }; @@ -71,6 +73,7 @@ pub fn workspaced_service() -> Router { .route("/list", get(list_apps)) .route("/list_search", get(list_search_apps)) .route("/get/p/*path", get(get_app)) + .route("/get/lite/*path", get(get_app_lite)) .route("/get/draft/*path", get(get_app_w_draft)) .route("/secret_of/*path", get(get_secret_id)) .route("/get/v/*id", get(get_app_by_id)) @@ -81,6 +84,7 @@ pub fn workspaced_service() -> Router { .route("/history/p/*path", get(get_app_history)) .route("/get_latest_version/*path", get(get_latest_version)) .route("/history_update/a/:id/v/:version", post(update_app_history)) + .route("/custom_path_exists/*custom_path", get(custom_path_exists)) } pub fn unauthed_service() -> Router { @@ -90,13 +94,17 @@ pub fn unauthed_service() -> Router { .route("/public_app/:secret", get(get_public_app_by_secret)) .route("/public_resource/*path", get(get_public_resource)) } - pub fn global_service() -> Router { Router::new() .route("/hub/list", get(list_hub_apps)) .route("/hub/get/:id", get(get_hub_app_by_id)) } +#[cfg(not(feature = "enterprise"))] +pub fn global_unauthed_service() -> Router { + Router::new() +} + #[derive(FromRow, Deserialize, Serialize)] pub struct ListableApp { pub id: i64, @@ -147,21 +155,26 @@ pub struct AppWithLastVersionAndStarred { pub starred: Option, } +#[cfg(feature = "enterprise")] +#[derive(Serialize, FromRow)] +pub struct AppWithLastVersionAndWorkspace { + #[sqlx(flatten)] + #[serde(flatten)] + pub app: AppWithLastVersion, + pub workspace_id: String, +} + #[derive(Serialize, Deserialize, FromRow)] pub struct AppWithLastVersionAndDraft { - pub id: i64, - pub path: String, - pub summary: String, - pub policy: sqlx::types::Json>, - pub versions: Vec, - pub value: sqlx::types::Json>, - pub created_by: String, - pub created_at: chrono::DateTime, - pub extra_perms: serde_json::Value, + #[sqlx(flatten)] + #[serde(flatten)] + pub app: AppWithLastVersion, #[serde(skip_serializing_if = "Option::is_none")] pub draft: Option>>, #[serde(skip_serializing_if = "Option::is_none")] pub draft_only: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub custom_path: Option, } #[derive(Serialize)] @@ -181,15 +194,16 @@ pub type StaticFields = HashMap>; pub type OneOfFields = HashMap>>; pub type AllowUserResources = Vec; -#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] #[serde(rename_all = "lowercase")] pub enum ExecutionMode { + #[default] Anonymous, Publisher, Viewer, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct PolicyTriggerableInputs { static_inputs: StaticFields, one_of_inputs: OneOfFields, @@ -205,7 +219,7 @@ pub struct S3Input { file_key_regex: String, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct Policy { pub on_behalf_of: Option, pub on_behalf_of_email: Option, @@ -229,6 +243,7 @@ pub struct CreateApp { pub policy: Policy, pub draft_only: Option, pub deployment_message: Option, + pub custom_path: Option, } #[derive(Deserialize)] @@ -238,6 +253,7 @@ pub struct EditApp { pub value: Option>>, pub policy: Option, pub deployment_message: Option, + pub custom_path: Option, } #[derive(Serialize, FromRow)] @@ -398,6 +414,33 @@ async fn get_app( Ok(Json(app)) } +async fn get_app_lite( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + + let app_o = sqlx::query_as::<_, AppWithLastVersion>( + "SELECT app.id, app.path, app.summary, app.versions, app.policy, + app.extra_perms, coalesce(app_version_lite.value::json, app_version.value) as value, + app_version.created_at, app_version.created_by, NULL as starred + FROM app, app_version + LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id + WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]", + ) + .bind(path.to_owned()) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + + tx.commit().await?; + + let app = not_found_if_none(app_o, "App", path)?; + Ok(Json(app)) +} + async fn get_app_w_draft( authed: ApiAuthed, Extension(user_db): Extension, @@ -408,7 +451,7 @@ async fn get_app_w_draft( let app_o = sqlx::query_as::<_, AppWithLastVersionAndDraft>( r#"SELECT app.id, app.path, app.summary, app.versions, app.policy, - app.extra_perms, app_version.value, + app.extra_perms, app_version.value, app.custom_path, app_version.created_at, app_version.created_by, app.draft_only, draft.value as "draft" from app @@ -515,6 +558,22 @@ async fn update_app_history( return Ok(()); } + +async fn custom_path_exists( + Extension(db): Extension, + Path((w_id, custom_path)): Path<(String, String)>, +) -> JsonResult { + let exists = + sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))", + custom_path, + if *CLOUD_HOSTED { Some(&w_id) } else { None } + ) + .fetch_one(&db) + .await?.unwrap_or(false); + Ok(Json(exists)) +} + async fn get_app_by_id( authed: ApiAuthed, Extension(user_db): Extension, @@ -555,8 +614,9 @@ async fn get_public_app_by_secret( let app_o = sqlx::query_as::<_, AppWithLastVersion>( "SELECT app.id, app.path, app.summary, app.versions, app.policy, - null as extra_perms, app_version.value, + null as extra_perms, coalesce(app_version_lite.value::json, app_version.value::json) as value, app_version.created_at, app_version.created_by from app, app_version + LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]") .bind(&id) .bind(&w_id) @@ -598,6 +658,7 @@ async fn get_public_app_by_secret( Ok(Json(app)) } + async fn get_public_resource( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, @@ -680,6 +741,26 @@ async fn create_app( ))); } + if let Some(custom_path) = &app.custom_path { + + require_admin(authed.is_admin, &authed.username)?; + + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))", + custom_path, + if *CLOUD_HOSTED { Some(&w_id) } else { None } + ) + .fetch_one(&mut *tx) + .await?.unwrap_or(false); + + if exists { + return Err(Error::BadRequest(format!( + "App with custom path {} already exists", + custom_path + ))); + } + } + sqlx::query!( "DELETE FROM draft WHERE path = $1 AND workspace_id = $2 AND typ = 'app'", &app.path, @@ -690,13 +771,14 @@ async fn create_app( let id = sqlx::query_scalar!( "INSERT INTO app - (workspace_id, path, summary, policy, versions, draft_only) - VALUES ($1, $2, $3, $4, '{}', $5) RETURNING id", + (workspace_id, path, summary, policy, versions, draft_only, custom_path) + VALUES ($1, $2, $3, $4, '{}', $5, $6) RETURNING id", w_id, app.path, app.summary, json!(app.policy), app.draft_only, + app.custom_path, ) .fetch_one(&mut *tx) .await?; @@ -899,7 +981,11 @@ async fn update_app( let mut tx = user_db.clone().begin(&authed).await?; - let npath = if ns.policy.is_some() || ns.path.is_some() || ns.summary.is_some() { + let npath = if ns.policy.is_some() + || ns.path.is_some() + || ns.summary.is_some() + || ns.custom_path.is_some() + { let mut sqlb = SqlBuilder::update_table("app"); sqlb.and_where_eq("path", "?".bind(&path)); sqlb.and_where_eq("workspace_id", "?".bind(&w_id)); @@ -932,6 +1018,29 @@ async fn update_app( sqlb.set_str("summary", nsummary); } + if let Some(ncustom_path) = &ns.custom_path { + + require_admin(authed.is_admin, &authed.username)?; + + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))", + ncustom_path, + if *CLOUD_HOSTED { Some(&w_id) } else { None }, + path, + w_id + ) + .fetch_one(&mut *tx) + .await?.unwrap_or(false); + + if exists { + return Err(Error::BadRequest(format!( + "App with custom path {} already exists", + ncustom_path + ))); + } + sqlb.set_str("custom_path", ncustom_path); + } + if let Some(mut npolicy) = ns.policy { npolicy.on_behalf_of = Some(username_to_permissioned_as(&authed.username)); npolicy.on_behalf_of_email = Some(authed.email.clone()); @@ -1067,6 +1176,10 @@ async fn update_app( #[derive(Debug, Deserialize, Clone)] pub struct ExecuteApp { + /// The app version to execute. Fallback to `path` if not provided. + pub version: Option, + /// The app script id (from the `app_script` table) to execute. + pub id: Option, pub args: HashMap>, // - script: script/ // - flow: flow/ @@ -1129,6 +1242,22 @@ async fn get_on_behalf_details_from_policy_and_authed( Ok((username, permissioned_as, email)) } +/// Convert the triggerables from the old format to the new format. +fn empty_triggerables(mut policy: Policy) -> Policy { + use std::mem::take; + if let Some(triggerables) = take(&mut policy.triggerables) { + let mut triggerables_v2 = take(&mut policy.triggerables_v2).unwrap_or_default(); + for (k, static_inputs) in triggerables.into_iter() { + triggerables_v2.insert( + k, + PolicyTriggerableInputs { static_inputs, ..Default::default() }, + ); + } + policy.triggerables_v2 = Some(triggerables_v2); + } + policy +} + async fn execute_component( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -1151,99 +1280,136 @@ async fn execute_component( }; let path = path.to_path(); + let (arc_policy, policy): (Arc, Policy); + let policy_triggerables_default = Default::default(); - let policy = match payload.clone() { + // Two cases here: + // 1. The component is executed from the editor (i.e. in "preview" mode), then: + // - The policy is set to default (in `Viewer` execution mode). + // - The policy triggerables are built by the frontend and retrieved from the request + // payload. + // - In case of inline script, the `RawCode` from the request is pushed as is to the + // job queue. + // 2. Otherwise (i.e. "run" mode): + // - The policy and triggerables are fetched from the database. + // - In case of inline script, if an entry exists in the `app_script` table, push + // an `AppScript` job payload, as in (.1) otherwise. + let (policy, policy_triggerables) = match payload { + // 1. "preview" mode. ExecuteApp { - force_viewer_static_fields: Some(static_fields), - force_viewer_one_of_fields: Some(one_of_fields), + force_viewer_static_fields: Some(static_inputs), + force_viewer_one_of_fields: Some(one_of_inputs), force_viewer_allow_user_resources: Some(allow_user_resources), .. - } => { - let mut hm = HashMap::new(); - - if let Some(path) = payload.path.clone() { - hm.insert( - format!("{}:{path}", payload.component), - PolicyTriggerableInputs { - static_inputs: static_fields, - one_of_inputs: one_of_fields, - allow_user_resources, - }, - ); - } else { - hm.insert( - format!( - "{}:{}", - payload.component, - digest(payload.raw_code.clone().unwrap().content.as_str()) - ), - PolicyTriggerableInputs { - static_inputs: static_fields, - one_of_inputs: one_of_fields, - allow_user_resources, - }, - ); - } - Policy { + } => ( + &Policy { execution_mode: ExecutionMode::Viewer, - triggerables: None, - triggerables_v2: Some(hm), - on_behalf_of: None, - on_behalf_of_email: None, - s3_inputs: None, - } - } + ..Default::default() + }, + &PolicyTriggerableInputs { + static_inputs, + one_of_inputs, + allow_user_resources, + }, + ), + // 2. "run" mode. _ => { - let policy_o = sqlx::query_scalar!( - "SELECT policy from app WHERE path = $1 AND workspace_id = $2", + // Policy is fetched from the database on app `path` and `workspace_id`. + let policy_fut = sqlx::query_scalar!( + "SELECT policy as \"policy: sqlx::types::Json>\" + FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1", path, - &w_id + &w_id, ) .fetch_optional(&db) - .await?; + .map_err(Into::::into) + .map(|policy_o| Result::Ok(not_found_if_none(policy_o?, "App", path)?)) + .map(|policy| Result::Ok(serde_json::from_str(policy?.get())?)) + .map_ok(empty_triggerables); - let policy = not_found_if_none(policy_o, "App", path)?; + // 1. The app `version` is provided: cache the fetched policy. + // 2. Otherwise, always fetch the policy from the database. + let policy = if let Some(id) = payload.version { + let cache = cache::anon!({ u64 => Arc } in "policy" <= 1000); + arc_policy = policy_fut + .map_ok(Arc::new) + .cached(cache, &(id as u64)) + .await?; + &*arc_policy + } else { + policy = policy_fut.await?; + &policy + }; - serde_json::from_value::(policy).map_err(to_anyhow)? + // Compute the path for the triggerables map: + // - flow: `flow/` + // - script: `script/` + // - inline script: `rawscript/` + let path = match &payload { + // flow or script: just use the `payload.path`. + ExecuteApp { path: Some(path), .. } => path, + // inline script: without entry in the `app_script` table. + ExecuteApp { raw_code: Some(raw_code), id: None, .. } => &digest(&raw_code.content), + // inline script: with an entry in the `app_script` table. + ExecuteApp { raw_code: Some(_), id: Some(id), .. } => { + let cache = cache::anon!({ u64 => Arc } in "appscriptpath" <= 10000); + // `id` is unique, cache the result. + &*sqlx::query_scalar!( + "SELECT format('rawscript/%s', code_sha256) as \"path!: String\" + FROM app_script WHERE id = $1 LIMIT 1", + id + ) + .fetch_one(&db) + .map_err(Into::::into) + .map_ok(Arc::new) + .cached(cache, &(*id as u64)) + .await? + } + _ => unreachable!(), + }; + + // Retrieve the triggerables from the policy on `path` or `:`. + let triggerables_v2 = policy + .triggerables_v2 + .as_ref() + .ok_or_else(|| Error::BadRequest(format!("Policy is missing triggerables")))?; + let policy_triggerables = triggerables_v2 + .get(path) // start with `path` in case we can avoid the next` format!`. + .or_else(|| triggerables_v2.get(&format!("{}:{}", payload.component, &path))) + .or(match policy.execution_mode { + ExecutionMode::Viewer => Some(&policy_triggerables_default), + _ => None, + }) + .ok_or_else(|| Error::BadRequest(format!("Path {path} forbidden by policy")))?; + + (policy, policy_triggerables) } }; let (username, permissioned_as, email) = get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; - let (job_payload, (args, job_id), tag) = match payload { - ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => { - let content = &raw_code.content; - let payload = JobPayload::Code(raw_code.clone()); - let path = digest(content); - let args = build_args( - policy, - &component, - path, - args, - opt_authed.as_ref(), - &user_db, - &db, - &w_id, - ) - .await?; - (payload, args, None) - } - ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => { - let (payload, tag) = get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?; - let args = build_args( - policy, - &component, - path.to_string(), - args, - opt_authed.as_ref(), - &user_db, - &db, - &w_id, - ) - .await?; - (payload, args, tag) - } + let (args, job_id) = build_args( + policy, + policy_triggerables, + payload.args, + opt_authed.as_ref(), + &user_db, + &db, + &w_id, + ) + .await?; + + let (job_payload, tag) = match (payload.path, payload.raw_code, payload.id) { + // flow or script: + (Some(path), None, None) => get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?, + // inline script: in "preview" mode or without entry in the `app_script` table. + (None, Some(raw_code), None) => (JobPayload::Code(raw_code), None), + // inline script: in "run" mode and with an entry in the `app_script` table. + (None, Some(RawCode { language, path, cache_ttl, .. }), Some(id)) => ( + JobPayload::AppScript { id: AppScriptId(id), cache_ttl, language, path }, + None, + ), _ => unreachable!(), }; let tx = windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()); @@ -1578,9 +1744,12 @@ async fn exists_app( } async fn build_args( - policy: Policy, - component: &str, - path: String, + policy: &Policy, + PolicyTriggerableInputs { + static_inputs, + one_of_inputs, + allow_user_resources, + }: &PolicyTriggerableInputs, mut args: HashMap>, authed: Option<&ApiAuthed>, user_db: &UserDB, @@ -1588,54 +1757,6 @@ async fn build_args( w_id: &str, ) -> Result<(PushArgsOwned, Option)> { let mut job_id: Option = None; - let key = format!("{}:{}", component, &path); - let (static_inputs, one_of_inputs, allow_user_resources) = match policy { - Policy { triggerables_v2: Some(t), .. } => { - let PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources } = t - .get(&key) - .or_else(|| t.get(&path)) - .map(|x| x.clone()) - .or_else(|| { - if matches!(policy.execution_mode, ExecutionMode::Viewer) { - Some(PolicyTriggerableInputs { - static_inputs: HashMap::new(), - one_of_inputs: HashMap::new(), - allow_user_resources: Vec::new(), - }) - } else { - None - } - }) - .ok_or_else(|| { - Error::BadRequest(format!("path {} is not allowed in the app policy", path)) - })?; - - (static_inputs, one_of_inputs, allow_user_resources) - } - Policy { triggerables: Some(t), .. } => { - let static_inputs = t - .get(&key) - .or_else(|| t.get(&path)) - .map(|x| x.clone()) - .or_else(|| { - if matches!(policy.execution_mode, ExecutionMode::Viewer) { - Some(HashMap::new()) - } else { - None - } - }) - .ok_or_else(|| { - Error::BadRequest(format!("path {} is not allowed in the app policy", path)) - })?; - - (static_inputs, HashMap::new(), Vec::new()) - } - _ => Err(Error::BadRequest(format!( - "Policy is missing triggerables for {}", - key - )))?, - }; - let mut safe_args = HashMap::>::new(); // tracing::error!("{:?}", allow_user_resources); @@ -1684,16 +1805,16 @@ async fn build_args( } for (k, v) in one_of_inputs { - if safe_args.contains_key(&k) { + if safe_args.contains_key(k) { continue; } - if let Some(arg_val) = args.get(&k) { + if let Some(arg_val) = args.get(k) { let arg_str = arg_val.get(); let options_str_vec = v.iter().map(|x| x.get()).collect::>(); if options_str_vec.contains(&arg_str) { safe_args.insert(k.to_string(), arg_val.clone()); - args.remove(&k); + args.remove(k); continue; } @@ -1704,7 +1825,7 @@ async fn build_args( .all(|x| options_str_vec.contains(&x.get())) { safe_args.insert(k.to_string(), arg_val.clone()); - args.remove(&k); + args.remove(k); continue; } } diff --git a/backend/windmill-api/src/apps_ee.rs b/backend/windmill-api/src/apps_ee.rs new file mode 100644 index 0000000000..a7737664b9 --- /dev/null +++ b/backend/windmill-api/src/apps_ee.rs @@ -0,0 +1,5 @@ +use axum::Router; + +pub fn global_unauthed_service() -> Router { + Router::new() +} diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 28f2523ad5..88243789b4 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -64,6 +64,8 @@ mod indexer_ee; mod inputs; mod integration; +#[cfg(feature = "enterprise")] +mod apps_ee; #[cfg(feature = "parquet")] mod job_helpers_ee; pub mod job_metrics; @@ -343,6 +345,17 @@ pub async fn run_server( ) .nest("/concurrency_groups", concurrency_groups::global_service()) .nest("/scripts_u", scripts::global_unauthed_service()) + .nest("/apps_u", { + #[cfg(feature = "enterprise")] + { + apps_ee::global_unauthed_service() + } + + #[cfg(not(feature = "enterprise"))] + { + Router::new() + } + }) .nest( "/w/:workspace_id/apps_u", apps::unauthed_service() diff --git a/backend/windmill-api/src/settings.rs b/backend/windmill-api/src/settings.rs index 61a7e46fef..c527eb481a 100644 --- a/backend/windmill-api/src/settings.rs +++ b/backend/windmill-api/src/settings.rs @@ -448,7 +448,7 @@ pub async fn get_critical_alerts( Extension(db): Extension, authed: ApiAuthed, Query(params): Query, -) -> JsonResult> { +) -> JsonResult { require_devops_role(&db, &authed.email).await?; crate::utils::get_critical_alerts(db, params, None).await diff --git a/backend/windmill-api/src/tracing_init.rs b/backend/windmill-api/src/tracing_init.rs index 60dab73810..d099eb1598 100644 --- a/backend/windmill-api/src/tracing_init.rs +++ b/backend/windmill-api/src/tracing_init.rs @@ -29,11 +29,13 @@ impl OnResponse for MyOnResponse { _span: &tracing::Span, ) { if *LOG_REQUESTS { - tracing::info!( - latency = latency.as_millis(), - status = response.status().as_u16(), - "response" - ) + let latency = latency.as_millis(); + let status = response.status().as_u16(); + if response.status().is_success() { + tracing::info!(latency = latency, status = status, "response") + } else { + tracing::error!(latency = latency, status = status, "response") + } } } } @@ -67,7 +69,7 @@ impl MakeSpan for MyMakeSpan { uri = %request.uri(), username = field::Empty, workspace_id = field::Empty, - trace_id = tracing_id, + traceId = tracing_id, email = field::Empty, ) } diff --git a/backend/windmill-api/src/utils.rs b/backend/windmill-api/src/utils.rs index cc6360b994..a003481c9a 100644 --- a/backend/windmill-api/src/utils.rs +++ b/backend/windmill-api/src/utils.rs @@ -10,6 +10,7 @@ use axum::{body::Body, response::Response}; use regex::Regex; use serde::Deserialize; use sqlx::{Postgres, Transaction}; +use windmill_common::worker::CLOUD_HOSTED; use windmill_common::{ auth::{is_devops_email, is_super_admin_email}, error::{self, Error}, @@ -206,11 +207,55 @@ pub async fn get_critical_alerts( db: DB, params: AlertQueryParams, workspace_id: Option, -) -> JsonResult> { +) -> JsonResult { + // Returning total rows and total pages let page = params.page.unwrap_or(1).max(1); let page_size = params.page_size.unwrap_or(10).min(100) as i64; let offset = ((page - 1) * page_size as i32) as i64; + // Count total rows + let total_rows = if let Some(workspace_id) = &workspace_id { + if params.acknowledged.is_none() { + sqlx::query_scalar!( + "SELECT COUNT(*) + FROM alerts + WHERE workspace_id = $1", + workspace_id + ) + .fetch_one(&db) + .await? + } else { + sqlx::query_scalar!( + "SELECT COUNT(*) + FROM alerts + WHERE workspace_id = $1 AND COALESCE(acknowledged_workspace, false) = $2", + workspace_id, + params.acknowledged + ) + .fetch_one(&db) + .await? + } + } else { + if params.acknowledged.is_none() { + sqlx::query_scalar!( + "SELECT COUNT(*) + FROM alerts" + ) + .fetch_one(&db) + .await? + } else { + sqlx::query_scalar!( + "SELECT COUNT(*) + FROM alerts + WHERE COALESCE(acknowledged, false) = $1", + params.acknowledged + ) + .fetch_one(&db) + .await? + } + }; + + // Fetch paginated rows let alerts = if let Some(workspace_id) = workspace_id { // `workspace_id` is provided => workspace admin if params.acknowledged.is_none() { @@ -278,7 +323,14 @@ pub async fn get_critical_alerts( } }; - Ok(Json(alerts)) + let total_rows = total_rows.unwrap_or(0); + let total_pages = ((total_rows as f64) / (page_size as f64)).ceil() as i64; + + Ok(Json(serde_json::json!({ + "alerts": alerts, + "total_rows": total_rows, + "total_pages": total_pages + }))) } #[cfg(feature = "enterprise")] @@ -292,12 +344,17 @@ pub async fn acknowledge_critical_alert( SET acknowledged = true, acknowledged_workspace = CASE - WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true - ELSE acknowledged_workspace + WHEN $3 THEN + CASE + WHEN $2::text IS NOT NULL AND workspace_id = $2 THEN true + ELSE acknowledged_workspace + END + ELSE true END WHERE id = $1", id, - workspace_id + workspace_id, + *CLOUD_HOSTED ) .execute(&db) .await?; @@ -320,12 +377,17 @@ pub async fn acknowledge_all_critical_alerts( SET acknowledged = true, acknowledged_workspace = CASE - WHEN $1::text IS NOT NULL THEN true - ELSE acknowledged_workspace + WHEN $2 THEN + CASE + WHEN $1::text IS NOT NULL THEN true + ELSE acknowledged_workspace + END + ELSE true END WHERE ($1::text IS NOT NULL AND workspace_id = $1) OR ($1::text IS NULL)", - workspace_id + workspace_id, + *CLOUD_HOSTED ) .execute(&db) .await?; diff --git a/backend/windmill-api/src/workspaces.rs b/backend/windmill-api/src/workspaces.rs index c6d6e5301c..bba06b238b 100644 --- a/backend/windmill-api/src/workspaces.rs +++ b/backend/windmill-api/src/workspaces.rs @@ -3111,7 +3111,7 @@ pub async fn get_critical_alerts( Path(w_id): Path, authed: ApiAuthed, Query(params): Query, -) -> JsonResult> { +) -> JsonResult { require_admin_or_devops(authed.is_admin, &authed.username, &authed.email, &db).await?; crate::utils::get_critical_alerts(db, params, Some(w_id)).await diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index 068668930d..99d5670f1b 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -13,6 +13,8 @@ flamegraph = ["dep:tracing-flame"] loki = ["dep:tracing-loki"] benchmark = [] parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"] +otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk", + "dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"] [lib] name = "windmill_common" @@ -62,6 +64,15 @@ windmill-macros.workspace = true semver.workspace = true croner = "2.0.6" quick_cache.workspace = true +pin-project-lite.workspace = true + +opentelemetry-semantic-conventions = { workspace = true, optional = true } +opentelemetry-otlp = { workspace = true, optional = true } +opentelemetry_sdk = { workspace = true, optional = true } +opentelemetry = { workspace = true, optional = true } +tracing-opentelemetry = { workspace = true, optional = true } +opentelemetry-appender-tracing = { workspace = true, optional = true } +tonic = { workspace = true, optional = true } [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemalloc-ctl = { optional = true, workspace = true } diff --git a/backend/windmill-common/src/apps.rs b/backend/windmill-common/src/apps.rs index acbf720f57..2948fd7cb7 100644 --- a/backend/windmill-common/src/apps.rs +++ b/backend/windmill-common/src/apps.rs @@ -6,7 +6,18 @@ * LICENSE-AGPL for a copy of the license. */ -use serde::Deserialize; +use serde::{Deserialize, Serialize}; + +/// Id in the `app_script` table. +#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)] +#[serde(transparent)] +pub struct AppScriptId(pub i64); + +impl Into for AppScriptId { + fn into(self) -> u64 { + self.0 as u64 + } +} #[derive(Deserialize)] pub struct ListAppQuery { diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index bb738bddf1..45fdd0bd82 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -1,25 +1,149 @@ use crate::error; +use std::future::Future; +use std::hash::Hash; use std::path::{Path, PathBuf}; -use quick_cache::sync::Cache; +use quick_cache::Equivalent; +use serde::{Deserialize, Serialize}; use sqlx::PgExecutor; +pub use const_format::concatcp; +pub use lazy_static::lazy_static; +pub use quick_cache::sync::Cache; + /// Cache directory for windmill server/worker(s). pub const CACHE_DIR: &str = "/tmp/windmill/cache/"; +/// A file-system backed concurrent cache. +pub struct FsBackedCache { + cache: Cache, + root: &'static str, +} + +impl FsBackedCache { + /// Create a new file-system backed cache with `items_capacity` capacity. + /// The cache will be stored in the `root` directory. + pub fn new(root: &'static str, items_capacity: usize) -> Self { + Self { cache: Cache::new(items_capacity), root } + } + + /// Gets or inserts an item in the cache with key `key`. + pub async fn get_or_insert_async<'a, Q, F>(&'a self, key: &Q, with: F) -> error::Result + where + Q: Hash + Equivalent + ToOwned + Copy + Into, + F: Future>, + { + self.cache + .get_or_insert_async( + key, + fs::import_or_insert_with(self.root, (*key).into(), with), + ) + .await + } +} + +/// Like [`lazy_static`]`, but for file-system backed caches. +/// +/// # Example +/// ```rust +/// use windmill_common::make_static; +/// +/// make_static! { +/// /// String cache with a maximum capacity of 1000 items stored in the +/// /// "subdirectory" directory. +/// static ref CACHE: { u64 => String } in "subdirectory" <= 1000; +/// /// Another cache. +/// static ref ANOTHER_CACHE: { u64 => Vec } in "another" <= 100; +/// } +/// ``` +#[macro_export] +macro_rules! make_static { + { $( $(#[$attr:meta])* static ref $name:ident: { $Key:ty => $Val:ty } in $root:literal <= $cap:literal; )+ } => { + $crate::cache::lazy_static! { + $( + $(#[$attr])* + static ref $name: $crate::cache::FsBackedCache<$Key, $Val> = + $crate::cache::FsBackedCache::new( + $crate::cache::concatcp!($crate::cache::CACHE_DIR, $root), + $cap + ); + )+ + } + }; +} + +// re-export: +pub use make_static; + +/// Create an anonymous file-system backed cache for one-time use. +/// +/// # Example +/// ```rust +/// use windmill_common::anon; +/// let cache = anon!({ u64 => String } in "subdirectory" <= 1000); +/// ``` +#[macro_export] +macro_rules! anon { + ({ $Key:ty => $Val:ty } in $root:literal <= $cap:literal) => {{ + $crate::cache::make_static! { + static ref __ANON__: { $Key => $Val } in $root <= $cap; + } + + &__ANON__ + }}; +} + +// re-export: +pub use anon; + +pub mod future { + use super::*; + + /// Extension trait for futures that can be cached. + pub trait FutureCachedExt: + Future> + Sized + { + /// Get or insert the future result in the cache. + /// + /// # Example + /// ```rust + /// use windmill_common::cache::{self, future::FutureCachedExt}; + /// + /// async { + /// let result = std::future::ready(Ok(42)) + /// .cached(cache::anon!({ u64 => u64 } in "test" <= 1), &42) + /// .await; + /// + /// assert_eq!(result.unwrap(), 42); + /// }; + /// ``` + fn cached( + self, + cache: &FsBackedCache, + key: &Q, + ) -> impl Future> + where + Q: Hash + Equivalent + ToOwned + Copy + Into, + { + cache.get_or_insert_async(key, self) + } + } + + impl> + Sized> + FutureCachedExt for F + { + } +} + pub mod flow { use super::*; use crate::flows::{FlowNodeId, FlowValue}; - /// Cache directory for windmill server/worker(s) flow nodes. - pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "flow"); - - lazy_static::lazy_static! { + make_static! { /// Flow node cache. - /// FIXME: This should be a static but [`Cache`] does not have a const constructor. /// FIXME: Use `Arc` for cheap cloning. - static ref CACHE: Cache = Cache::new(1000); + static ref CACHE: { FlowNodeId => Val } in "flow" <= 1000; } /// Flow node cache value. @@ -34,24 +158,36 @@ pub mod flow { /// If not present, import from the file-system cache or fetch it from the database and write /// it to the file system and cache. /// This should be preferred over fetching the database directly. - pub async fn fetch_script(e: impl PgExecutor<'_>, node: FlowNodeId) - -> error::Result<(Option, String)> - { - fetch(e, node).await.and_then(|Val { lock, code, .. }| Ok((lock, code.ok_or_else(|| { - error::Error::InternalErr(format!("Flow node ({:x}) isn't a script node.", node.0)) - })?))) + pub async fn fetch_script( + e: impl PgExecutor<'_>, + node: FlowNodeId, + ) -> error::Result<(Option, String)> { + fetch(e, node).await.and_then(|Val { lock, code, .. }| { + Ok(( + lock, + code.ok_or_else(|| { + error::Error::InternalErr(format!( + "Flow node ({:x}) isn't a script node.", + node.0 + )) + })?, + )) + }) } /// Fetch the flow node flow value referenced by `node` from the cache. /// If not present, import from the file-system cache or fetch it from the database and write /// it to the file system and cache. /// This should be preferred over fetching the database directly. - pub async fn fetch_flow(e: impl PgExecutor<'_>, node: FlowNodeId) - -> error::Result - { - fetch(e, node).await.and_then(|Val { flow, .. }| flow.ok_or_else(|| { - error::Error::InternalErr(format!("Flow node ({:x}) isn't a flow value node.", node.0)) - })) + pub async fn fetch_flow(e: impl PgExecutor<'_>, node: FlowNodeId) -> error::Result { + fetch(e, node).await.and_then(|Val { flow, .. }| { + flow.ok_or_else(|| { + error::Error::InternalErr(format!( + "Flow node ({:x}) isn't a flow value node.", + node.0 + )) + }) + }) } /// Fetch the flow node referenced by `node` from the cache. @@ -62,9 +198,8 @@ pub mod flow { // If not present, `get_or_insert_async` will lock the key until the future completes, // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. - CACHE.get_or_insert_async( - &node, - fs::import_or_insert_with(CACHE_DIR, node.0 as u64, async { + CACHE + .get_or_insert_async(&node, async { sqlx::query!( "SELECT \ lock AS \"lock: String\", \ @@ -76,18 +211,24 @@ pub mod flow { .fetch_one(e) .await .map_err(Into::into) - .and_then(|r| Ok(Val { - lock: r.lock.and_then(|x| if x.is_empty() { None } else { Some(x) }), - code: r.code, - flow: match r.flow { - None => None, - Some(flow) => serde_json::from_str(&flow).map_err(|err| { - error::Error::InternalErr(format!("Unable to parse flow value: {err:?}")) - })?, - } - })) + .and_then(|r| { + Ok(Val { + lock: r + .lock + .and_then(|x| if x.is_empty() { None } else { Some(x) }), + code: r.code, + flow: match r.flow { + None => None, + Some(flow) => serde_json::from_str(&flow).map_err(|err| { + error::Error::InternalErr(format!( + "Unable to parse flow value: {err:?}" + )) + })?, + }, + }) + }) }) - ).await + .await } // ---------------------------------------------------------------------------------------------- @@ -130,7 +271,11 @@ pub mod flow { match item { Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())), Item::Code => Ok(self.code.as_ref().map(|s| s.as_bytes().to_vec())), - Item::Flow => Ok(self.flow.as_ref().map(|f| serde_json::to_vec(f)).transpose()?), + Item::Flow => Ok(self + .flow + .as_ref() + .map(|f| serde_json::to_vec(f)) + .transpose()?), } } } @@ -140,14 +285,10 @@ pub mod script { use super::*; use crate::scripts::{ScriptHash, ScriptLang}; - /// Cache directory for windmill server/worker(s) scripts. - pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "script"); - - lazy_static::lazy_static! { + make_static! { /// Scripts cache. - /// FIXME: This should be a static but [`Cache`] does not have a const constructor. /// FIXME: Use `Arc` for cheap cloning. - static ref CACHE: Cache = Cache::new(1000); + static ref CACHE: { ScriptHash => Val } in "script" <= 1000; } /// Script cache value. @@ -164,15 +305,16 @@ pub mod script { /// If not present, import from the file-system cache or fetch it from the database and write /// it to the file system and cache. /// This should be preferred over fetching the database directly. - pub async fn fetch(e: impl PgExecutor<'_>, hash: ScriptHash, workspace_id: &str) - -> error::Result - { + pub async fn fetch( + e: impl PgExecutor<'_>, + hash: ScriptHash, + workspace_id: &str, + ) -> error::Result { // If not present, `get_or_insert_async` will lock the key until the future completes, // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. - CACHE.get_or_insert_async( - &hash, - fs::import_or_insert_with(CACHE_DIR, hash.0 as u64, async { + CACHE + .get_or_insert_async(&hash, async { sqlx::query!( "SELECT \ lock AS \"lock: String\", \ @@ -188,15 +330,16 @@ pub mod script { .await .map_err(Into::into) .map(|r| Val { - lock: r.lock.and_then(|x| if x.is_empty() { None } else { Some(x) }), + lock: r + .lock + .and_then(|x| if x.is_empty() { None } else { Some(x) }), code: r.code, language: r.language, envs: r.envs, codebase: r.codebase, }) }) - ) - .await + .await } // ---------------------------------------------------------------------------------------------- @@ -230,7 +373,104 @@ pub mod script { match item { Item::Lock => self.lock = Some(String::from_utf8(data)?), Item::Code => self.code = String::from_utf8(data)?, - Item::Info => (self.language, self.envs, self.codebase) = serde_json::from_slice(&data)?, + Item::Info => { + (self.language, self.envs, self.codebase) = serde_json::from_slice(&data)? + } + } + Ok(()) + } + + fn export(&self, item: Self::Item) -> error::Result>> { + match item { + Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())), + Item::Code => Ok(Some(self.code.as_bytes().to_vec())), + Item::Info => Ok(Some(serde_json::to_vec(&( + &self.language, + &self.envs, + &self.codebase, + ))?)), + } + } + } +} + +pub mod app { + use super::*; + use crate::apps::AppScriptId; + + make_static! { + /// App scripts cache. + /// FIXME: Use `Arc` for cheap cloning. + static ref CACHE: { AppScriptId => Val } in "app" <= 1000; + } + + /// App app script cache value. + #[derive(Debug, Clone, Default)] + pub struct Val { + pub lock: Option, + pub code: String, + } + + /// Fetch the app script referenced by `id` from the cache. + /// If not present, import from the file-system cache or fetch it from the database and write + /// it to the file system and cache. + /// This should be preferred over fetching the database directly. + pub async fn fetch_script( + e: impl PgExecutor<'_>, + id: AppScriptId, + ) -> error::Result<(Option, String)> { + // If not present, `get_or_insert_async` will lock the key until the future completes, + // so only one thread will be able to fetch the data from the database and write it to + // the file system and cache, hence no race on the file system. + CACHE + .get_or_insert_async(&id, async { + sqlx::query!( + "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", + id.0, + ) + .fetch_one(e) + .await + .map_err(Into::into) + .map(|r| Val { + lock: r + .lock + .and_then(|x| if x.is_empty() { None } else { Some(x) }), + code: r.code, + }) + }) + .await + .map(|Val { lock, code }| (lock, code)) + } + + // ---------------------------------------------------------------------------------------------- + // impl `fs::Bundle` for `Val`. + + #[derive(Copy, Clone)] + pub enum Item { + Lock, + Code, + } + + impl fs::Item for Item { + fn path(&self, root: &Path) -> PathBuf { + match self { + Item::Lock => root.join("lock.txt"), + Item::Code => root.join("code.txt"), + } + } + } + + impl fs::Bundle for Val { + type Item = Item; + + fn items() -> &'static [Self::Item] { + &[Item::Lock, Item::Code] + } + + fn import(&mut self, item: Self::Item, data: Vec) -> error::Result<()> { + match item { + Item::Lock => self.lock = Some(String::from_utf8(data)?), + Item::Code => self.code = String::from_utf8(data)?, } Ok(()) } @@ -239,7 +479,6 @@ pub mod script { match item { Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())), Item::Code => Ok(Some(self.code.as_bytes().to_vec())), - Item::Info => Ok(Some(serde_json::to_vec(&(&self.language, &self.envs, &self.codebase))?)), } } } @@ -248,8 +487,6 @@ pub mod script { mod fs { use super::*; - use std::future::Future; - use std::fs::{self, OpenOptions}; use std::io::{Read, Write}; @@ -272,8 +509,7 @@ mod fs { } /// Import or insert a bundle within the given combination of `{root}/{key}/`. - pub async fn import_or_insert_with(root: &str, key: u64, f: F) - -> error::Result + pub async fn import_or_insert_with(root: &str, key: u64, f: F) -> error::Result where T: Bundle, F: Future>, @@ -287,8 +523,9 @@ mod fs { let mut data = T::default(); for item in T::items() { let mut buf = vec![]; - let Ok(mut file) = OpenOptions::new().read(true).open(item.path(&path)) - else { continue }; + let Ok(mut file) = OpenOptions::new().read(true).open(item.path(&path)) else { + continue; + }; file.read_to_end(&mut buf)?; data.import(*item, buf)?; } @@ -299,7 +536,7 @@ mod fs { Ok(data) => return Ok(data), Err(err) => tracing::warn!( "Failed to import from file-system, fetch source..: {path:?}: {err:?}" - ) + ), } } // Cache path doesn't exist or import failed, generate the content. @@ -308,9 +545,13 @@ mod fs { fs::create_dir_all(&path)?; // Write the generated data to the file. for item in T::items() { - let Some(buf) = data.export(*item)? - else { continue }; - let mut file = OpenOptions::new().write(true).create(true).open(item.path(&path))?; + let Some(buf) = data.export(*item)? else { + continue; + }; + let mut file = OpenOptions::new() + .write(true) + .create(true) + .open(item.path(&path))?; file.write_all(&buf)?; } tracing::debug!("Exported to file-system: {:?}", path); @@ -324,4 +565,29 @@ mod fs { } Ok(data) } + + // Auto-implement `Bundle` for all `serde` serializable types. + + impl Item for () { + fn path(&self, root: &Path) -> PathBuf { + root.join("self.json") + } + } + + impl Deserialize<'de> + Serialize + Default> Bundle for T { + type Item = (); + + fn items() -> &'static [Self::Item] { + &[()] + } + + fn import(&mut self, _: Self::Item, data: Vec) -> error::Result<()> { + *self = serde_json::from_slice(&data)?; + Ok(()) + } + + fn export(&self, _: Self::Item) -> error::Result>> { + Ok(Some(serde_json::to_vec(self)?)) + } + } } diff --git a/backend/windmill-common/src/flow_status.rs b/backend/windmill-common/src/flow_status.rs index a5c509d6ef..50c7cf0e61 100644 --- a/backend/windmill-common/src/flow_status.rs +++ b/backend/windmill-common/src/flow_status.rs @@ -17,7 +17,7 @@ use crate::flows::FlowValue; const MINUTES: Duration = Duration::from_secs(60); const HOURS: Duration = MINUTES.saturating_mul(60); -pub const MAX_RETRY_ATTEMPTS: u16 = 1000; +pub const MAX_RETRY_ATTEMPTS: u32 = u32::MAX; pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6); pub fn is_retry_default(v: &RetryStatus) -> bool { @@ -48,7 +48,7 @@ pub struct FlowStatus { #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct RetryStatus { - pub fail_count: u16, + pub fail_count: u32, pub failed_jobs: Vec, } diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 97429cd081..29edd4db30 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -147,7 +147,7 @@ impl Retry { /// Takes the number of previous retries and returns the interval until the next retry if any. /// /// May return [`Duration::ZERO`] to retry immediately. - pub fn interval(&self, previous_attempts: u16, silent: bool) -> Option { + pub fn interval(&self, previous_attempts: u32, silent: bool) -> Option { let Self { constant, exponential } = self; if previous_attempts < constant.attempts { @@ -178,7 +178,7 @@ impl Retry { self.constant.attempts != 0 || self.exponential.attempts != 0 } - pub fn max_attempts(&self) -> u16 { + pub fn max_attempts(&self) -> u32 { self.constant .attempts .saturating_add(self.exponential.attempts) @@ -194,7 +194,7 @@ impl Retry { #[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] #[serde(default)] pub struct ConstantDelay { - pub attempts: u16, + pub attempts: u32, pub seconds: u16, } @@ -202,7 +202,7 @@ pub struct ConstantDelay { #[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] #[serde(default)] pub struct ExponentialDelay { - pub attempts: u16, + pub attempts: u32, pub multiplier: u16, pub seconds: u16, pub random_factor: Option, // percentage, defaults to 0 for no jitter @@ -407,6 +407,12 @@ pub enum InputTransform { #[serde(transparent)] pub struct FlowNodeId(pub i64); +impl Into for FlowNodeId { + fn into(self) -> u64 { + self.0 as u64 + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Branch { #[serde(skip_serializing_if = "Option::is_none")] @@ -713,10 +719,14 @@ pub async fn resolve_maybe_value( workspace_id: &str, with_code: bool, maybe: Option, - value_mut: impl FnOnce(&mut T) -> Option<&mut Json>> + value_mut: impl FnOnce(&mut T) -> Option<&mut Json>>, ) -> Result, Error> { - let Some(mut container) = maybe else { return Ok(None); }; - let Some(value) = value_mut(&mut container) else { return Ok(Some(container)); }; + let Some(mut container) = maybe else { + return Ok(None); + }; + let Some(value) = value_mut(&mut container) else { + return Ok(Some(container)); + }; resolve_value(e, workspace_id, &mut value.0, with_code).await?; Ok(Some(container)) } @@ -728,8 +738,9 @@ pub async fn resolve_value( value: &mut Box, with_code: bool, ) -> Result<(), Error> { - let mut val = serde_json::from_str::(value.get()) - .map_err(|err| Error::InternalErr(format!("resolve: Failed to parse flow value: {}", err)))?; + let mut val = serde_json::from_str::(value.get()).map_err(|err| { + Error::InternalErr(format!("resolve: Failed to parse flow value: {}", err)) + })?; for module in &mut val.modules { resolve_module(e, workspace_id, &mut module.value, with_code).await?; } @@ -746,16 +757,29 @@ pub async fn resolve_module( ) -> Result<(), Error> { use FlowModuleValue::*; - let mut val = serde_json::from_str::(value.get()) - .map_err(|err| Error::InternalErr(format!("resolve: Failed to parse flow module value: {}", err)))?; + let mut val = serde_json::from_str::(value.get()).map_err(|err| { + Error::InternalErr(format!( + "resolve: Failed to parse flow module value: {}", + err + )) + })?; match &mut val { FlowScript { .. } => { // In order to avoid an unnecessary `.clone()` of `val`, take ownership of it's content // using `std::mem::replace`. let FlowScript { - input_transforms, id, tag, language, - custom_concurrency_key, concurrent_limit, concurrency_time_window_s, is_trigger - } = std::mem::replace(&mut val, Identity) else { unreachable!() }; + input_transforms, + id, + tag, + language, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + is_trigger, + } = std::mem::replace(&mut val, Identity) + else { + unreachable!() + }; // Load script lock file and code content. let (lock, content) = if !with_code { (Some("...".to_string()), "...".to_string()) @@ -763,22 +787,44 @@ pub async fn resolve_module( cache::flow::fetch_script(e, id).await? }; val = RawScript { - input_transforms, content, lock, path: None, tag, language, custom_concurrency_key, - concurrent_limit, concurrency_time_window_s, is_trigger + input_transforms, + content, + lock, + path: None, + tag, + language, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + is_trigger, }; - }, - ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => { + } + ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => { resolve_modules(e, workspace_id, modules, modules_node.take(), with_code).await?; - }, + } BranchOne { branches, default, default_node } => { resolve_modules(e, workspace_id, default, default_node.take(), with_code).await?; for branch in branches { - resolve_modules(e, workspace_id, &mut branch.modules, branch.modules_node.take(), with_code).await?; + resolve_modules( + e, + workspace_id, + &mut branch.modules, + branch.modules_node.take(), + with_code, + ) + .await?; } - }, + } BranchAll { branches, .. } => { for branch in branches { - resolve_modules(e, workspace_id, &mut branch.modules, branch.modules_node.take(), with_code).await?; + resolve_modules( + e, + workspace_id, + &mut branch.modules, + branch.modules_node.take(), + with_code, + ) + .await?; } } _ => {} @@ -801,7 +847,13 @@ pub async fn resolve_modules( .map(|flow| flow.modules)?; } for module in modules.iter_mut() { - Box::pin(resolve_module(e, workspace_id, &mut module.value, with_code)).await?; + Box::pin(resolve_module( + e, + workspace_id, + &mut module.value, + with_code, + )) + .await?; } Ok(()) } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 7b932d8f50..9dbfd03b0e 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -35,8 +35,9 @@ pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; +pub const OTEL_SETTING: &str = "otel"; -pub const ENV_SETTINGS: [&str; 51] = [ +pub const ENV_SETTINGS: [&str; 54] = [ "DISABLE_NSJAIL", "MODE", "NUM_WORKERS", @@ -88,4 +89,7 @@ pub const ENV_SETTINGS: [&str; 51] = [ "WORKER_GROUP", "SAML_METADATA", "INSTANCE_IS_DEV", + "OTEL_METRICS", + "OTEL_TRACING", + "OTEL_LOGS", ]; diff --git a/backend/windmill-common/src/job_metrics.rs b/backend/windmill-common/src/job_metrics.rs index c0c2300fed..6577e34d6e 100644 --- a/backend/windmill-common/src/job_metrics.rs +++ b/backend/windmill-common/src/job_metrics.rs @@ -1,4 +1,4 @@ -use crate::{db::DB, error}; +use crate::{db::DB, error, utils::WarnAfterExt}; use serde::{Deserialize, Serialize}; use uuid::Uuid; @@ -90,6 +90,7 @@ pub async fn register_metric_for_job( .bind(timeseries_int) .bind(timeseries_float) .execute(db) + .warn_after_seconds(1) .await?; Ok(metric_id) diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 5b687b54fb..c8f8470b69 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -14,6 +14,7 @@ pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; pub const PREPROCESSOR_FAKE_ENTRYPOINT: &str = "__WM_PREPROCESSOR"; use crate::{ + apps::AppScriptId, error::{self, to_anyhow, Error}, flow_status::{FlowStatus, RestartedFrom}, flows::{FlowNodeId, FlowValue, Retry}, @@ -41,6 +42,7 @@ pub enum JobKind { DeploymentCallback, FlowScript, FlowNode, + AppScript, } #[derive(sqlx::FromRow, Debug, Serialize, Clone)] @@ -278,6 +280,12 @@ pub enum JobPayload { id: FlowNodeId, // flow_node(id). path: String, // flow node inner path (e.g. `outer/branchall-42`). }, + AppScript { + id: AppScriptId, // app_script(id). + path: Option, + language: ScriptLang, + cache_ttl: Option, + }, Code(RawCode), Dependencies { path: String, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index ad47b198af..4140f6e58b 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -36,20 +36,20 @@ pub mod job_s3_helpers_ee; pub mod jobs; pub mod more_serde; pub mod oauth2; +pub mod otel_ee; pub mod queue; pub mod s3_helpers; pub mod schedule; pub mod scripts; pub mod server; pub mod stats_ee; +pub mod tracing_init; pub mod users; pub mod utils; pub mod variables; pub mod worker; pub mod workspaces; -pub mod tracing_init; - pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50; pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5; pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; @@ -87,6 +87,12 @@ lazy_static::lazy_static! { .unwrap_or_else(|| SocketAddr::from(([0, 0, 0, 0], *METRICS_PORT))); pub static ref METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("METRICS_PORT").is_ok() || std::env::var("METRICS_ADDR").is_ok()); + + pub static ref OTEL_METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_METRICS").is_ok()); + pub static ref OTEL_TRACING_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_TRACING").is_ok()); + pub static ref OTEL_LOGS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_LOGS").is_ok()); + + pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false); pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false); diff --git a/backend/windmill-common/src/otel_ee.rs b/backend/windmill-common/src/otel_ee.rs new file mode 100644 index 0000000000..f3ada162f6 --- /dev/null +++ b/backend/windmill-common/src/otel_ee.rs @@ -0,0 +1,58 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use crate::{jobs::QueuedJob, utils::Mode}; +use uuid::Uuid; + +pub fn set_span_parent(_span: &tracing::Span, _rj: &Uuid) {} + +#[cfg(not(all(feature = "otel", feature = "enterprise")))] +pub(crate) type OtelProvider = Option<()>; + +#[cfg(all(feature = "otel", feature = "enterprise"))] +pub(crate) type OtelProvider = Option; + +#[cfg(not(feature = "otel"))] +pub fn otel_ctx() -> () {} + +#[cfg(feature = "otel")] +#[inline(always)] +pub fn otel_ctx() -> opentelemetry::Context { + opentelemetry::Context::current() +} + +#[cfg(not(feature = "otel"))] +impl FutureExt for T {} + +#[cfg(not(feature = "otel"))] +pub trait FutureExt: Sized { + fn with_context(self, _otel_cx: ()) -> Self { + self + } +} + +use tracing_subscriber::EnvFilter; + +pub(crate) fn init_logs_bridge(_mode: &Mode, _hostname: &str, _env: &str) -> Option { + None +} + +#[cfg(all(feature = "otel", feature = "enterprise"))] +pub(crate) fn init_otlp_tracer( + _mode: &Mode, + _hostname: &str, + _env: &str, +) -> Option { + None +} + +pub(crate) fn init_meter_provider(_mode: &Mode, _hostname: &str, _env: &str) -> OtelProvider { + None +} + +pub fn add_root_flow_job_to_otlp(_queued_job: &QueuedJob, _success: bool) {} diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 1a11b5a701..9d51bbb830 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -75,6 +75,12 @@ impl ScriptLang { #[sqlx(transparent)] pub struct ScriptHash(pub i64); +impl Into for ScriptHash { + fn into(self) -> u64 { + self.0 as u64 + } +} + #[derive(PartialEq, sqlx::Type)] #[sqlx(transparent, no_pg_array)] pub struct ScriptHashes(pub Vec); diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 13b345e36e..aa1e409dd1 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -7,13 +7,23 @@ */ use const_format::concatcp; + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; +use tracing::Event; use tracing_appender::non_blocking::{NonBlockingBuilder, WorkerGuard}; +use tracing_subscriber::layer::Context; use tracing_subscriber::{ + filter::Targets, fmt::{format, Layer}, prelude::*, EnvFilter, }; +use crate::utils::Mode; + fn json_layer() -> Layer> { tracing_subscriber::fmt::layer() .json() @@ -34,7 +44,11 @@ pub const LOGS_SERVICE: &str = "logs/services/"; pub const TMP_WINDMILL_LOGS_SERVICE: &str = concatcp!("/tmp/windmill/", LOGS_SERVICE); -pub fn initialize_tracing(hostname: &str) -> WorkerGuard { +pub fn initialize_tracing( + hostname: &str, + mode: &Mode, + environment: &str, +) -> (WorkerGuard, crate::otel_ee::OtelProvider) { let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()); if std::env::var("RUST_LOG").is_ok_and(|x| x == "debug" || x == "info") { @@ -44,7 +58,17 @@ pub fn initialize_tracing(hostname: &str) -> WorkerGuard { ) } - let env_filter = EnvFilter::from_default_env(); + let meter_provider = crate::otel_ee::init_meter_provider(mode, hostname, environment); + + #[cfg(all(feature = "otel", feature = "enterprise"))] + let opentelemetry = crate::otel_ee::init_otlp_tracer(mode, hostname, environment) + .map(|x| tracing_opentelemetry::layer().with_tracer(x)); + + #[cfg(not(all(feature = "otel", feature = "enterprise")))] + let opentelemetry: Option = None; + + let logs_bridge = crate::otel_ee::init_logs_bridge(&mode, hostname, environment); + use tracing_appender::rolling::{RollingFileAppender, Rotation}; let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname); @@ -61,39 +85,59 @@ pub fn initialize_tracing(hostname: &str) -> WorkerGuard { .finish(file_appender); let stdout_and_log_file_writer = std::io::stdout.and(log_file_writer); - let ts_base = tracing_subscriber::registry().with(env_filter); + // let job_logs_filter = tracing_subscriber::filter::Targets::new() + // .with_target("windmill:job_log", tracing::Level::TRACE); - #[cfg(feature = "loki")] - let ts_base = { - let (layer, task) = tracing_loki::builder() - .build_url(reqwest::Url::parse("http://127.0.0.1:3100").unwrap()) - .expect("build loki url"); - tokio::spawn(task); - ts_base.with(layer) - }; + let env_filter = EnvFilter::builder() + .with_default_directive(tracing::level_filters::LevelFilter::ERROR.into()) + .from_env_lossy(); + + let ts_base = tracing_subscriber::registry().with(env_filter); match *JSON_FMT { true => ts_base + .with(logs_bridge) + .with(opentelemetry) + // .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap())) .with( json_layer() .with_writer(stdout_and_log_file_writer) - .flatten_event(true), + .flatten_event(true) + .with_filter( + Targets::new() + .with_target( + "windmill:job_log", + tracing::level_filters::LevelFilter::OFF, + ) + .with_default(tracing::level_filters::LevelFilter::INFO), + ), ) .with(CountingLayer::new()) .init(), false => ts_base + .with(logs_bridge) + .with(opentelemetry) + // .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap())) .with( compact_layer() .with_writer(stdout_and_log_file_writer) .with_ansi(style.to_lowercase() != "never") .with_file(true) .with_line_number(true) - .with_target(false), + .with_target(false) + .with_filter( + Targets::new() + .with_target( + "windmill:job_log", + tracing::level_filters::LevelFilter::OFF, + ) + .with_default(tracing::level_filters::LevelFilter::INFO), + ), ) .with(CountingLayer::new()) .init(), } - _guard + (_guard, meter_provider) } #[cfg(feature = "flamegraph")] @@ -112,13 +156,6 @@ pub fn setup_flamegraph() -> impl Drop { _guard } -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; -use tracing::Event; -use tracing_subscriber::layer::Context; - lazy_static::lazy_static! { pub static ref LOG_COUNTING_BY_MIN: Arc>> = Arc::new(RwLock::new(HashMap::new())); } @@ -144,22 +181,6 @@ impl CountingLayer { } } -// impl CountingLayer { -// pub fn new() -> Self { -// CountingLayer { counter: Arc::new(Mutex::new(LogCounter::new())) } -// } - -// pub fn get_counts(&self) -> (usize, usize) { -// let counter = self.counter.lock().unwrap(); -// (counter.non_error_count, counter.error_count) -// } - -// pub fn reset_counts(&self) { -// let mut counter = self.counter.lock().unwrap(); -// counter.reset(); -// } -// } - pub const LOG_TIMESTAMP_FMT: &str = "%Y-%m-%d-%H-%M"; impl tracing_subscriber::Layer for CountingLayer diff --git a/backend/windmill-common/src/utils.rs b/backend/windmill-common/src/utils.rs index 08a0c32e3b..c13f96e2de 100644 --- a/backend/windmill-common/src/utils.rs +++ b/backend/windmill-common/src/utils.rs @@ -34,8 +34,8 @@ pub const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version"); use crate::CRITICAL_ALERT_MUTE_UI_ENABLED; +use std::panic::{self, AssertUnwindSafe, Location}; use std::sync::atomic::Ordering; -use std::panic::{self, AssertUnwindSafe}; use crate::worker::CLOUD_HOSTED; @@ -78,14 +78,18 @@ pub fn require_admin(is_admin: bool, username: &str) -> Result<()> { } } -pub async fn require_admin_or_devops(is_admin: bool, username: &str, email: &str, db: &DB) -> Result<()> { +pub async fn require_admin_or_devops( + is_admin: bool, + username: &str, + email: &str, + db: &DB, +) -> Result<()> { if !is_admin { if !is_devops_email(db, email).await? { return Err(Error::RequireAdmin(username.to_string())); } } Ok(()) - } pub fn hostname() -> String { @@ -94,7 +98,7 @@ pub fn hostname() -> String { .to_str() .map(|x| x.to_string()) .unwrap_or_else(|| rd_string(5)) - }) + }) } pub fn paginate(pagination: Pagination) -> (usize, usize) { @@ -440,9 +444,7 @@ impl ScheduleType { Some("v2") | Some(_) => { // Use Croner for v2 let schedule_type_result = panic::catch_unwind(AssertUnwindSafe(|| { - Cron::new(schedule_str) - .with_seconds_optional() - .parse() + Cron::new(schedule_str).with_seconds_optional().parse() })) .map_err(|_| { tracing::error!( @@ -478,8 +480,14 @@ impl ScheduleType { } if let Err(e) = result { - tracing::error!("An error occurred while finding the next occurrence: {:?}", e); - return Err(Error::BadRequest(format!("cron: error during find_next_occurrence: {:?}", e))); + tracing::error!( + "An error occurred while finding the next occurrence: {:?}", + e + ); + return Err(Error::BadRequest(format!( + "cron: error during find_next_occurrence: {:?}", + e + ))); } } @@ -526,3 +534,78 @@ impl ScheduleType { Ok(events) } } + +use std::future::Future; +use std::pin::Pin; +use std::task::{Context as TContext, Poll}; +use tokio::time::{self, Duration, Sleep}; + +use pin_project_lite::pin_project; + +pub trait WarnAfterExt: Future + Sized { + /// Warns if the future takes longer than the specified number of seconds to complete. + #[track_caller] + fn warn_after_seconds(self, seconds: u8) -> WarnAfterFuture { + let caller = Location::caller(); + let location = format!("{}:{}", caller.file(), caller.line()); + WarnAfterFuture { + future: self, + timeout: time::sleep(Duration::from_secs(seconds as u64)), + warned: false, + start_time: std::time::Instant::now(), + location: location, + seconds, + } + } +} + +// Blanket implementation for all futures. +impl WarnAfterExt for F {} + +pin_project! { + /// A future that wraps another future and prints a warning if it takes too long. + pub struct WarnAfterFuture { + #[pin] + future: F, + #[pin] + timeout: Sleep, + warned: bool, + location: String, + start_time: std::time::Instant, + seconds: u8, + } +} + +impl Future for WarnAfterFuture { + type Output = F::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut TContext<'_>) -> Poll { + let this = self.project(); + + // Poll the timeout future to check if it has elapsed. + if !*this.warned { + if this.timeout.poll(cx).is_ready() { + tracing::warn!(location = this.location, "SLOW_QUERY: query to db taking longer than expected (> {} seconds). This is a sign the database is under heavy load, query is too heavy or database is undersized", + this.seconds, + ); + *this.warned = true; + } + } + + // Poll the wrapped future. + match this.future.poll(cx) { + Poll::Ready(output) => { + if *this.warned { + let elapsed = this.start_time.elapsed(); + tracing::warn!( + location = this.location, + "SLOW_QUERY: completed with total duration: {:.2?}", + elapsed + ); + } + Poll::Ready(output) + } + Poll::Pending => Poll::Pending, + } + } +} diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 9a4064ff1d..73205f6620 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -17,7 +17,9 @@ use std::{ use tokio::sync::RwLock; use windmill_macros::annotations; -use crate::{error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB}; +use crate::{ + error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB, +}; lazy_static::lazy_static! { pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string()); @@ -352,6 +354,11 @@ pub struct SqlAnnotations { pub return_last_result: bool, } +#[annotations("#")] +pub struct BashAnnotations { + pub docker: bool, +} + pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) { if tokio::fs::metadata(&bin_path).await.is_ok() { (true, format!("loaded from local cache: {}\n", bin_path)) @@ -577,8 +584,10 @@ pub fn get_windmill_memory_usage() -> Option { } } -pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(executor: E) -> bool { - use crate::utils::{GIT_VERSION, GIT_SEM_VERSION}; +pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>( + executor: E, +) -> bool { + use crate::utils::{GIT_SEM_VERSION, GIT_VERSION}; // fetch all pings with a different version than self from the last 5 minutes. let pings = sqlx::query_scalar!( diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index 7ac744a885..8f94de918a 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -43,4 +43,5 @@ bigdecimal.workspace = true axum.workspace = true serde_urlencoded.workspace = true regex.workspace = true -backon.workspace = true \ No newline at end of file +backon.workspace = true +opentelemetry.workspace = true diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 2390ca47bb..ef37834400 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -32,15 +32,14 @@ use sqlx::{types::Json, FromRow, Pool, Postgres, Transaction}; #[cfg(feature = "benchmark")] use std::time::Instant; use tokio::{sync::RwLock, time::sleep}; -use tracing::{instrument, Instrument}; use ulid::Ulid; use uuid::Uuid; use windmill_audit::audit_ee::{audit_log, AuditAuthor}; use windmill_audit::ActionKind; use windmill_common::{ - cache, auth::{fetch_authed_from_permissioned_as, permissioned_as_to_username}, + cache, db::{Authed, UserDB}, error::{self, to_anyhow, Error}, flow_status::{ @@ -57,7 +56,7 @@ use windmill_common::{ schedule::Schedule, scripts::{get_full_hub_script_by_path, ScriptHash, ScriptLang}, users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL}, - utils::{not_found_if_none, report_critical_error, StripPath}, + utils::{not_found_if_none, report_critical_error, StripPath, WarnAfterExt}, worker::{ to_raw_value, CLOUD_HOSTED, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, DISABLE_FLOW_SCRIPT, MIN_VERSION_IS_AT_LEAST_1_427, MIN_VERSION_IS_AT_LEAST_1_432, NO_LOGS, @@ -318,6 +317,7 @@ pub async fn append_logs( workspace.as_ref(), ) .execute(db.borrow()) + .warn_after_seconds(1) .await { tracing::error!(%job_id, %err, "error updating logs for large_log job {job_id}: {err}"); @@ -453,7 +453,6 @@ where } } -#[instrument(level = "trace", skip_all)] pub async fn add_completed_job_error( db: &Pool, queued_job: &QueuedJob, @@ -509,7 +508,6 @@ lazy_static::lazy_static! { pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok(); } -#[instrument(level = "trace", skip_all, name = "add_completed_job")] pub async fn add_completed_job( db: &Pool, queued_job: &QueuedJob, @@ -642,9 +640,7 @@ pub async fn add_completed_job( .fetch_one(&mut *tx) .await .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e:#}")))?; - // tracing::error!("2 {:?}", start.elapsed()); - // add_time!(bench, "add_completed_job query END"); if !queued_job.is_flow_step { if _duration > 500 @@ -1258,7 +1254,6 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> Ok(()) } -#[instrument(level = "trace", skip_all)] pub async fn handle_maybe_scheduled_job<'c>( db: &Pool, job: &QueuedJob, @@ -2428,7 +2423,6 @@ async fn extract_result_from_job_result( } } -#[instrument(level = "trace", skip_all)] pub async fn delete_job<'c>( mut tx: Transaction<'c, Postgres>, w_id: &str, @@ -3228,6 +3222,26 @@ pub async fn push<'c, 'd>( None, ) }, + JobPayload::AppScript { + id, // app_script(id). + path, + language, + cache_ttl, + } => ( + Some(id.0), + path, + None, + JobKind::AppScript, + None, + None, + Some(language), + None, + None, + None, + cache_ttl, + None, + None, + ), JobPayload::ScriptHub { path } => { if path == "hub/7771/slack" || path == "hub/7836/slack" { permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string(); @@ -3851,6 +3865,7 @@ pub async fn push<'c, 'd>( tag, ) .execute(&mut *tx) + .warn_after_seconds(1) .await?; let (raw_code, raw_lock, raw_flow) = if !*MIN_VERSION_IS_AT_LEAST_1_427.read().await { @@ -3901,6 +3916,7 @@ pub async fn push<'c, 'd>( final_priority, ) .fetch_one(&mut *tx) + .warn_after_seconds(1) .await .map_err(|e| Error::InternalErr(format!("Could not insert into queue {job_id} with tag {tag}, schedule_path {schedule_path:?}, script_path: {script_path:?}, email {email}, workspace_id {workspace_id}: {e:#}")))?; @@ -3986,6 +4002,7 @@ pub async fn push<'c, 'd>( JobKind::DeploymentCallback => "jobs.run.deployment_callback", JobKind::FlowScript => "jobs.run.flow_script", JobKind::FlowNode => "jobs.run.flow_node", + JobKind::AppScript => "jobs.run.app_script", }; let audit_author = if format!("u/{user}") != permissioned_as && user != permissioned_as { @@ -4015,7 +4032,6 @@ pub async fn push<'c, 'd>( script_path.as_ref().map(|x| x.as_str()), Some(hm), ) - .instrument(tracing::info_span!("job_run", email = &email)) .await?; } diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index cb64757a7f..f4df376e6d 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -18,7 +18,10 @@ parquet = ["windmill-common/parquet", "dep:object_store"] flow_testing = [] cloud = [] sqlx = [] -deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", "dep:deno_ast", "dep:deno_tls"] +deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", + "dep:deno_ast", "dep:deno_tls", "dep:deno_permissions"] +otel = ["windmill-common/otel", "dep:opentelemetry"] +dind = ["dep:bollard"] [dependencies] windmill-queue.workspace = true @@ -68,6 +71,8 @@ deno_url = { workspace = true, optional = true } deno_core = { workspace = true, optional = true } deno_ast = { workspace = true, optional = true } deno_tls = { workspace = true, optional = true } +deno_permissions = { workspace = true, optional = true } + postgres-native-tls.workspace = true native-tls.workspace = true mysql_async.workspace = true @@ -92,6 +97,9 @@ yaml-rust.workspace = true swc_ecma_parser.workspace = true backon.workspace = true +opentelemetry = { workspace = true, optional = true } +bollard = { workspace = true, optional = true } + [build-dependencies] deno_fetch = { workspace = true, optional = true } deno_webidl = { workspace = true, optional = true } @@ -102,4 +110,6 @@ deno_url = { workspace = true, optional = true } deno_core = { workspace = true, optional = true } deno_ast = { workspace = true, optional = true } deno_tls = { workspace = true, optional = true } +deno_permissions = { workspace = true, optional = true } + zstd.workspace = true diff --git a/backend/windmill-worker/build.rs b/backend/windmill-worker/build.rs index 6bbc3fd75f..e48ecddb35 100644 --- a/backend/windmill-worker/build.rs +++ b/backend/windmill-worker/build.rs @@ -25,7 +25,7 @@ impl FetchPermissions for PermissionsContainer { &mut self, _url: &deno_core::url::Url, _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { + ) -> Result<(), deno_permissions::PermissionCheckError> { unreachable!("snapshotting") } @@ -34,7 +34,7 @@ impl FetchPermissions for PermissionsContainer { &mut self, _p: &'a std::path::Path, _api_name: &str, - ) -> Result, deno_core::error::AnyError> { + ) -> Result, deno_permissions::PermissionCheckError> { unreachable!("snapshotting") } } @@ -53,7 +53,7 @@ impl NetPermissions for PermissionsContainer { &mut self, _p: &'a str, _api_name: &str, - ) -> Result { + ) -> Result { unreachable!("snapshotting") } @@ -61,7 +61,7 @@ impl NetPermissions for PermissionsContainer { &mut self, _p: &'a str, _api_name: &str, - ) -> Result { + ) -> Result { unreachable!("snapshotting") } @@ -69,7 +69,7 @@ impl NetPermissions for PermissionsContainer { &mut self, _host: &(T, Option), _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { + ) -> Result<(), deno_permissions::PermissionCheckError> { unreachable!("snapshotting") } @@ -77,7 +77,7 @@ impl NetPermissions for PermissionsContainer { &mut self, _: &'a Path, _: &str, - ) -> Result, deno_core::anyhow::Error> { + ) -> Result, deno_permissions::PermissionCheckError> { todo!() } } @@ -95,6 +95,7 @@ fn main() { println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); let exts = vec![ + // deno_telemetry::deno_telemetry::init_ops_and_esm(), deno_webidl::deno_webidl::init_ops_and_esm(), deno_url::deno_url::init_ops_and_esm(), deno_console::deno_console::init_ops_and_esm(), @@ -116,10 +117,10 @@ fn main() { deno_core::snapshot::CreateSnapshotOptions { cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"), startup_snapshot: None, + extension_transpiler: None, extensions: exts, with_runtime_cb: None, skip_op_registration: false, - extension_transpiler: None, }, None, ) diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index e51c6b9d5b..c2d917ad86 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -433,6 +433,10 @@ fi .args(cmd_args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + ansible_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + start_child_process(ansible_cmd, ANSIBLE_PLAYBOOK_PATH.as_str()).await? }; @@ -565,13 +569,9 @@ async fn create_file_resources( get_resource_or_variable_content(client, &file_res.resource_path, job_id.to_string()) .await?; let path = file_res.target_path.clone(); - let validated_path = write_file_at_user_defined_location( - job_dir, - path.as_str(), - &r, - file_res.mode, - ) - .map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?; + let validated_path = + write_file_at_user_defined_location(job_dir, path.as_str(), &r, file_res.mode) + .map_err(|e| anyhow!("Couldn't write text file at {}: {}", path, e))?; nsjail_mounts.push( define_nsjail_mount(job_dir, &validated_path) diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 2582f98b58..b6279ccf97 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -1,14 +1,30 @@ use std::{collections::HashMap, fs, process::Stdio}; +#[cfg(feature = "dind")] +use bollard::container::{ + KillContainerOptions, RemoveContainerOptions, StatsOptions, StopContainerOptions, +}; +#[cfg(feature = "dind")] +use futures::{stream, StreamExt, TryStreamExt}; use regex::Regex; use serde_json::{json, value::RawValue}; use sqlx::types::Json; use tokio::process::Command; + +#[cfg(feature = "dind")] +use uuid::Uuid; use windmill_common::{ error::Error, jobs::QueuedJob, worker::{to_raw_value, write_file}, }; + +#[cfg(feature = "dind")] +use windmill_common::DB; + +#[cfg(feature = "dind")] +use windmill_common::error::to_anyhow; + use windmill_queue::{append_logs, CanceledBy}; lazy_static::lazy_static! { @@ -22,6 +38,9 @@ lazy_static::lazy_static! { static ref RE_POWERSHELL_IMPORTS: Regex = Regex::new(r#"^Import-Module\s+(?:-Name\s+)?"?([^-\s"]+)"?"#).unwrap(); } +#[cfg(feature = "dind")] +use crate::handle_child::run_future_with_polling_update_job_poller; + use crate::{ common::{ build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process, @@ -54,8 +73,14 @@ pub async fn handle_bash_job( worker_name: &str, envs: HashMap, occupancy_metrics: &mut OccupancyMetrics, + _killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, ) -> Result, Error> { - let logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string(); + let annotation = windmill_common::worker::BashAnnotations::parse(&content); + + let mut logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string(); + if annotation.docker { + logs1.push_str("docker mode\n"); + } append_logs(&job.id, &job.workspace_id, logs1, db).await; write_file(job_dir, "main.sh", &format!("set -e\n{content}"))?; @@ -193,6 +218,22 @@ exit $exit_status ) .await?; + #[cfg(feature = "dind")] + if annotation.docker { + return handle_docker_job( + job.id, + &job.workspace_id, + db, + job.timeout, + mem_peak, + canceled_by, + worker_name, + occupancy_metrics, + _killpill_rx, + ) + .await; + } + let result_json_path = format!("{job_dir}/result.json"); if let Ok(metadata) = tokio::fs::metadata(&result_json_path).await { if metadata.len() > 0 { @@ -214,6 +255,7 @@ exit $exit_status .await? .trim() .to_string(); + return Ok(to_raw_value(&json!(result))); } @@ -222,6 +264,190 @@ exit $exit_status ))) } +#[cfg(feature = "dind")] +async fn handle_docker_job( + job_id: Uuid, + workspace_id: &str, + db: &DB, + job_timeout: Option, + mem_peak: &mut i32, + canceled_by: &mut Option, + worker_name: &str, + occupancy_metrics: &mut OccupancyMetrics, + killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, +) -> Result, Error> { + let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow)?; + + let container_id = job_id.to_string(); + let inspected = client.inspect_container(&container_id, None).await; + + if inspected.is_err() || inspected.unwrap().state.is_none() { + return Ok(to_raw_value(&format!( + "Container not found at {job_id}, you must use --name and not --rm it" + ))); + } + + let wait_f = async { + let wait = client + .wait_container::(&container_id, None) + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("Error waiting for container: {:?}", e); + anyhow::anyhow!("Error waiting for container") + })?; + let waited = wait.first().map(|x| x.status_code); + Ok(waited) + }; + + let ncontainer_id = container_id.to_string(); + let w_id = workspace_id.to_string(); + let j_id = job_id.clone(); + let db2 = db.clone(); + let (tx, mut rx) = tokio::sync::broadcast::channel::<()>(1); + + let mut killpill_rx = killpill_rx.resubscribe(); + let logs = tokio::spawn(async move { + let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow); + if let Ok(client) = client { + let mut log_stream = client.logs( + &ncontainer_id, + Some(bollard::container::LogsOptions { + follow: true, + stdout: true, + stderr: true, + tail: "all", + ..Default::default() + }), + ); + loop { + tokio::select! { + log = log_stream.next() => { + match log { + Some(Ok(log)) => { + append_logs(&j_id, w_id.clone(), log.to_string(), db2.clone()).await; + } + Some(Err(e)) => { + tracing::error!("Error getting logs: {:?}", e); + } + _ => { + tracing::error!("End of stream"); + return + } + }; + }, + _ = killpill_rx.recv() => { + tracing::error!("killing container after receving killpill"); + if let Err(e) = client + .stop_container(&ncontainer_id, Some(StopContainerOptions { t: 3 })) + .await + { + tracing::error!("Error stopping container: {:?}", e); + } + return + }, + _ = rx.recv() => { + return + } + } + } + } + }); + + let mem_client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow); + let ncontainer_id = container_id.clone(); + let result = run_future_with_polling_update_job_poller( + job_id, + job_timeout, + db, + mem_peak, + canceled_by, + wait_f, + worker_name, + workspace_id, + &mut Some(occupancy_metrics), + Box::pin(match mem_client { + Ok(client) => client + .stats( + &ncontainer_id, + Some(StatsOptions { stream: true, one_shot: false }), + ) + .map(|x| { + x.map(|x| x.memory_stats.usage.map(|m| m / 1024).unwrap_or_default() as i32) + .unwrap_or_default() + }) + .boxed(), + _ => stream::once(async { 0 }).boxed(), + }), + ) + .await; + + if let Err(e) = result { + if !logs.is_finished() { + let _ = tx.send(()); + let _ = logs.await; + } + if container_is_alive(&client, &container_id).await { + kill_container(&client, &container_id, "SIGINT").await; + if container_is_alive(&client, &container_id).await { + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + if let Err(e) = client + .stop_container(&container_id, Some(StopContainerOptions { t: 3 })) + .await + { + tracing::error!("Error stopping container: {:?}", e); + } + } + } + + return Err(e); + } + + if let Err(e) = client + .remove_container( + &container_id, + Some(RemoveContainerOptions { force: true, ..Default::default() }), + ) + .await + { + tracing::error!("Error removing container: {:?}", e); + } + + let result = result.unwrap(); + + return Ok(to_raw_value(&json!(format!( + "Docker exit status: {}", + result + .map(|x| x.to_string()) + .unwrap_or_else(|| "none".to_string()) + )))); +} + +#[cfg(feature = "dind")] +async fn kill_container(client: &bollard::Docker, container_id: &str, signal: &str) { + if let Err(e) = client + .kill_container(&container_id, Some(KillContainerOptions { signal })) + .await + { + tracing::error!("Error killing container with signal {signal}: {:?}", e); + } +} + +#[cfg(feature = "dind")] +async fn container_is_alive(client: &bollard::Docker, container_id: &str) -> bool { + let inspect = client.inspect_container(container_id, None).await; + if let Ok(inspect) = inspect { + let r = inspect + .state + .map(|x| x.running.unwrap_or_default()) + .unwrap_or_default(); + tracing::error!("Container {container_id} is alive: {r}"); + r + } else { + false + } +} + fn raw_to_string(x: &str) -> String { match serde_json::from_str::(x) { Ok(serde_json::Value::String(x)) => x, @@ -522,7 +748,8 @@ $env:PSModulePath = \"{};$PSModulePathBackup\"", std::env::var("PATHEXT").unwrap_or_else(|_| { String::from(".COM;.EXE;.BAT;.CMD;.VBS;.VBE;.JS;.JSE;.WSF;.WSH;.MSC;.CPL") }), - ); + ) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); } cmd.spawn()? diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index 2438392680..30c0f1d575 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -366,6 +366,7 @@ pub async fn do_bigquery( worker_name, &job.workspace_id, &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await?; diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index a7d92874e1..707a668b84 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -1522,6 +1522,15 @@ pub async fn get_common_bun_proc_envs(base_internal_url: Option<&str>) -> HashMa bun_envs.insert(String::from("NODE_PATH"), node_path.to_string()); } + #[cfg(windows)] + { + bun_envs.insert("SystemRoot".to_string(), crate::SYSTEM_ROOT.to_string()); + bun_envs.insert( + "USERPROFILE".to_string(), + crate::USERPROFILE_ENV.to_string(), + ); + } + return bun_envs; } diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index 31d8ecc4dd..f36954819d 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -866,11 +866,11 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error { #[cfg(windows)] let err_msg = "program not found"; - if err - .to_string() - .contains(&err_msg) - { - return Error::InternalErr(format!("Executable {executable} not found on worker. PATH: {}", *PATH_ENV)); + if err.to_string().contains(&err_msg) { + return Error::InternalErr(format!( + "Executable {executable} not found on worker. PATH: {}", + *PATH_ENV + )); } return err; } diff --git a/backend/windmill-worker/src/deno_executor.rs b/backend/windmill-worker/src/deno_executor.rs index 88f6733d1a..54a2631ccd 100644 --- a/backend/windmill-worker/src/deno_executor.rs +++ b/backend/windmill-worker/src/deno_executor.rs @@ -84,6 +84,15 @@ async fn get_common_deno_proc_envs( if DENO_TLS_CA_STORE.len() > 0 { deno_envs.insert(String::from("DENO_TLS_CA_STORE"), DENO_TLS_CA_STORE.clone()); } + + #[cfg(windows)] + { + deno_envs.insert("SystemRoot".to_string(), crate::SYSTEM_ROOT.to_string()); + deno_envs.insert( + "USERPROFILE".to_string(), + crate::USERPROFILE_ENV.to_string(), + ); + } return deno_envs; } diff --git a/backend/windmill-worker/src/go_executor.rs b/backend/windmill-worker/src/go_executor.rs index 470e647383..ea06f82c5a 100644 --- a/backend/windmill-worker/src/go_executor.rs +++ b/backend/windmill-worker/src/go_executor.rs @@ -194,6 +194,10 @@ func Run(req Req) (interface{{}}, error){{ .args(vec!["build", "main.go"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()); + + #[cfg(windows)] + build_go_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + let build_go_process = start_child_process(build_go_cmd, GO_PATH.as_str()).await?; handle_child( &job.id, @@ -293,6 +297,9 @@ func Run(req Req) (interface{{}}, error){{ run_go.env("GOPROXY", goproxy); } + #[cfg(windows)] + run_go.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + run_go.stdout(Stdio::piped()).stderr(Stdio::piped()); start_child_process(run_go, compiled_executable_name).await? }; diff --git a/backend/windmill-worker/src/graphql_executor.rs b/backend/windmill-worker/src/graphql_executor.rs index 8fb8cf13b9..67bb6a2c7b 100644 --- a/backend/windmill-worker/src/graphql_executor.rs +++ b/backend/windmill-worker/src/graphql_executor.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use anyhow::anyhow; -use futures::TryStreamExt; +use futures::{stream, TryStreamExt}; use serde_json::{json, value::RawValue}; use sqlx::types::Json; use windmill_common::jobs::QueuedJob; @@ -154,6 +154,7 @@ pub async fn do_graphql( worker_name, &job.workspace_id, &mut Some(occupation_metrics), + Box::pin(stream::once(async { 0 })), ) .await?; diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 4d09c36abc..f2fead39c9 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -87,7 +87,7 @@ async fn kill_process_tree(pid: Option) -> Result<(), String> { /// - update the `last_line` and `logs` strings with the program output /// - update "queue"."last_ping" every five seconds /// - kill process if we exceed timeout or "queue"."canceled" is set -#[tracing::instrument(level = "trace", skip_all)] +#[tracing::instrument(name="run_subprocess", level = "info", skip_all, fields(otel.name = %child_name))] pub async fn handle_child( job_id: &Uuid, db: &Pool, @@ -126,7 +126,7 @@ pub async fn handle_child( let (tx, rx) = broadcast::channel::<()>(3); let mut rx2 = tx.subscribe(); - let output = child_joined_output_stream(&mut child); + let output = child_joined_output_stream(&mut child, job_id.clone()); let job_id = job_id.clone(); @@ -137,7 +137,9 @@ pub async fn handle_child( db, mem_peak, canceled_by_ref, - || get_mem_peak(pid, nsjail), + Box::pin(stream::unfold((), move |_| async move { + Some((get_mem_peak(pid, nsjail).await, ())) + })), worker, w_id, rx, @@ -435,8 +437,6 @@ pub async fn handle_child( } } - - async fn get_mem_peak(pid: Option, nsjail: bool) -> i32 { if pid.is_none() { return -1; @@ -488,7 +488,7 @@ async fn get_mem_peak(pid: Option, nsjail: bool) -> i32 { } } -pub async fn run_future_with_polling_update_job_poller( +pub async fn run_future_with_polling_update_job_poller( job_id: Uuid, timeout: Option, db: &DB, @@ -498,9 +498,11 @@ pub async fn run_future_with_polling_update_job_poller( worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + get_mem: S, ) -> error::Result where Fut: Future>, + S: stream::Stream + Unpin, { let (tx, rx) = broadcast::channel::<()>(3); @@ -509,7 +511,7 @@ where db, mem_peak, canceled_by_ref, - || async { 0 }, + get_mem, worker_name, w_id, rx, @@ -550,20 +552,19 @@ pub enum UpdateJobPollingExit { AlreadyCompleted, } -pub async fn update_job_poller( +pub async fn update_job_poller( job_id: Uuid, db: &DB, mem_peak: &mut i32, canceled_by_ref: &mut Option, - get_mem: F, + mut get_mem: S, worker_name: &str, w_id: &str, mut rx: broadcast::Receiver<()>, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> UpdateJobPollingExit where - F: Fn() -> Fut, - Fut: Future, + S: stream::Stream + Unpin, { let update_job_interval = Duration::from_millis(500); @@ -608,7 +609,7 @@ where .expect("update worker ping"); } } - let current_mem = get_mem().await; + let current_mem = get_mem.next().await.unwrap_or(0); if current_mem > *mem_peak { *mem_peak = current_mem } @@ -679,6 +680,7 @@ where /// builds a stream joining both stdout and stderr each read line by line fn child_joined_output_stream( child: &mut Child, + job_id: Uuid, ) -> impl stream::FusedStream> { let stderr = child .stderr @@ -692,24 +694,24 @@ fn child_joined_output_stream( let stdout = BufReader::new(stdout).lines(); let stderr = BufReader::new(stderr).lines(); - stream::select(lines_to_stream(stderr, true), lines_to_stream(stdout, false)) + stream::select( + lines_to_stream(stderr, true, job_id.clone()), + lines_to_stream(stdout, false, job_id), + ) } pub fn lines_to_stream( mut lines: tokio::io::Lines, stderr: bool, + job_id: Uuid, ) -> impl futures::Stream> { stream::poll_fn(move |cx| { std::pin::Pin::new(&mut lines) .poll_next_line(cx) - .map(|result| { - process_streaming_log_lines(result, stderr) - }) + .map(|result| process_streaming_log_lines(result, stderr, &job_id)) }) } - - pub fn process_status(status: ExitStatus) -> error::Result<()> { if status.success() { Ok(()) diff --git a/backend/windmill-worker/src/job_logger_ee.rs b/backend/windmill-worker/src/job_logger_ee.rs index 5414ccd496..310419e3d1 100644 --- a/backend/windmill-worker/src/job_logger_ee.rs +++ b/backend/windmill-worker/src/job_logger_ee.rs @@ -34,6 +34,7 @@ pub(crate) async fn default_disk_log_storage( pub(crate) fn process_streaming_log_lines( r: Result, io::Error>, _stderr: bool, + _job_id: &Uuid, ) -> Option> { r.transpose() } diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index b4b84e2e2b..e5fd20bffd 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -7,14 +7,7 @@ */ #[cfg(feature = "deno_core")] -use std::{ - borrow::Cow, - cell::RefCell, - env, - io::{self, BufReader}, - path::PathBuf, - rc::Rc, -}; +use std::{borrow::Cow, cell::RefCell, env, path::PathBuf, rc::Rc}; use std::{collections::HashMap, sync::Arc}; @@ -32,7 +25,7 @@ use deno_fetch::FetchPermissions; #[cfg(feature = "deno_core")] use deno_net::NetPermissions; #[cfg(feature = "deno_core")] -use deno_tls::{rustls::RootCertStore, rustls_pemfile}; +use deno_tls::rustls::RootCertStore; #[cfg(feature = "deno_core")] use deno_web::{BlobStore, TimersPermission}; #[cfg(feature = "deno_core")] @@ -68,35 +61,35 @@ pub struct IdContext { pub previous_id: String, } -#[cfg(feature = "deno_core")] -pub struct ContainerRootCertStoreProvider { - root_cert_store: RootCertStore, -} +// #[cfg(feature = "deno_core")] +// pub struct ContainerRootCertStoreProvider { +// root_cert_store: RootCertStore, +// } -#[cfg(feature = "deno_core")] -impl ContainerRootCertStoreProvider { - fn new() -> ContainerRootCertStoreProvider { - return ContainerRootCertStoreProvider { - root_cert_store: deno_tls::create_default_root_cert_store(), - }; - } +// #[cfg(feature = "deno_core")] +// impl ContainerRootCertStoreProvider { +// fn new() -> ContainerRootCertStoreProvider { +// return ContainerRootCertStoreProvider { +// root_cert_store: deno_tls::create_default_root_cert_store(), +// }; +// } - fn add_certificate(&mut self, cert_path: String) -> io::Result<()> { - let cert_file = std::fs::File::open(cert_path)?; - let mut reader = BufReader::new(cert_file); - let pem_file = rustls_pemfile::certs(&mut reader).collect::, _>>()?; +// fn add_certificate(&mut self, cert_path: String) -> io::Result<()> { +// let cert_file = std::fs::File::open(cert_path)?; +// let mut reader = BufReader::new(cert_file); +// let pem_file = rustls_pemfile::certs(&mut reader).collect::, _>>()?; - self.root_cert_store.add_parsable_certificates(pem_file); - Ok(()) - } -} +// self.root_cert_store.add_parsable_certificates(pem_file); +// Ok(()) +// } +// } -#[cfg(feature = "deno_core")] -impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider { - fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> { - Ok(&self.root_cert_store) - } -} +// #[cfg(feature = "deno_core")] +// impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider { +// fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> { +// Ok(&self.root_cert_store) +// } +// } #[cfg(feature = "deno_core")] pub struct PermissionsContainer; @@ -108,7 +101,7 @@ impl FetchPermissions for PermissionsContainer { &mut self, _url: &deno_core::url::Url, _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { + ) -> Result<(), deno_permissions::PermissionCheckError> { Ok(()) } @@ -117,7 +110,7 @@ impl FetchPermissions for PermissionsContainer { &mut self, p: &'a std::path::Path, _api_name: &str, - ) -> Result, anyhow::Error> { + ) -> Result, deno_permissions::PermissionCheckError> { Ok(Cow::Borrowed(p)) } } @@ -136,7 +129,7 @@ impl NetPermissions for PermissionsContainer { &mut self, p: &'a str, _api_name: &str, - ) -> Result { + ) -> Result { Ok(PathBuf::from(p)) } @@ -144,7 +137,7 @@ impl NetPermissions for PermissionsContainer { &mut self, p: &'a str, _api_name: &str, - ) -> Result { + ) -> Result { Ok(PathBuf::from(p)) } @@ -152,7 +145,7 @@ impl NetPermissions for PermissionsContainer { &mut self, _host: &(T, Option), _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { + ) -> Result<(), deno_permissions::PermissionCheckError> { Ok(()) } @@ -160,7 +153,7 @@ impl NetPermissions for PermissionsContainer { &mut self, p: &'a std::path::Path, _api_name: &str, - ) -> Result, AnyError> { + ) -> Result, deno_permissions::PermissionCheckError> { Ok(Cow::Borrowed(p)) } } @@ -669,9 +662,12 @@ pub fn transpile_ts(expr: String) -> anyhow::Result { text: deno_core::ModuleCodeString::from(expr).into(), })?; Ok(parsed - .transpile(&Default::default(), &Default::default())? + .transpile( + &Default::default(), + &Default::default(), + &Default::default(), + )? .into_source() - .into_string()? .text) } @@ -814,13 +810,7 @@ pub async fn eval_fetch_timeout( let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() }; let fetch_options = deno_fetch::Options { - root_cert_store_provider: if let Some(cert_path) = env::var("DENO_CERT").ok() { - let mut cert_store_provider = ContainerRootCertStoreProvider::new(); - cert_store_provider.add_certificate(cert_path)?; - Some(Arc::new(cert_store_provider)) - } else { - None - }, + root_cert_store_provider: None, user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()), proxy: ann.proxy.map(|x| deno_tls::Proxy { url: x.0, @@ -930,6 +920,7 @@ pub async fn eval_fetch_timeout( worker_name, w_id, &mut Some(occupation_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await .map_err(|e| { diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index 7c4724418c..b97560e225 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -160,6 +160,7 @@ pub async fn do_mssql( worker_name, &job.workspace_id, &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await?; diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 3ed6a9bf34..b98ec365ee 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -298,6 +298,7 @@ pub async fn do_mysql( worker_name, &job.workspace_id, &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await?; diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 57d7f48905..2ad6eb4038 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -355,6 +355,7 @@ pub async fn do_postgresql( worker_name, &job.workspace_id, &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await?; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 4127af7f7b..6df2c0bc0b 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -318,7 +318,7 @@ pub async fn uv_pip_compile( if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { args.extend(["--cert", cert_path]); } - tracing::debug!("uv args: {:?}", args); + tracing::error!("uv args: {:?}", args); #[cfg(windows)] let uv_cmd = "uv"; @@ -329,7 +329,7 @@ pub async fn uv_pip_compile( let mut child_cmd = Command::new(uv_cmd); child_cmd .current_dir(job_dir) - .args(args) + .args(&args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); let child_process = start_child_process(child_cmd, uv_cmd).await?; @@ -350,7 +350,7 @@ pub async fn uv_pip_compile( occupancy_metrics, ) .await - .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed.\n\ncommand: {uv_cmd} {}\n\n{e:?}", args.join(" "))))?; } let path_lock = format!("{job_dir}/requirements.txt"); @@ -749,8 +749,10 @@ mount {{ .stdout(Stdio::piped()) .stderr(Stdio::piped()); - #[cfg(windows)] - python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + #[cfg(windows)] { + python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); + python_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + } start_child_process(python_cmd, PYTHON_PATH.as_str()).await? }; @@ -1277,6 +1279,7 @@ async fn spawn_uv_install( .envs(envs) .envs(PROXY_ENVS.clone()) .env("SystemRoot", SYSTEM_ROOT.as_str()) + .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) .env( "TMP", std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index ea8a883407..69684eb271 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -1,3 +1,6 @@ +#[cfg(feature = "otel")] +use opentelemetry::trace::FutureExt; + use serde::Serialize; use sqlx::{types::Json, Pool, Postgres}; use std::{ @@ -7,6 +10,9 @@ use std::{ Arc, }, }; +use tracing::{field, Instrument}; +#[cfg(not(feature = "otel"))] +use windmill_common::otel_ee::FutureExt; use uuid::Uuid; @@ -14,6 +20,7 @@ use windmill_common::{ add_time, error::{self, Error}, jobs::{JobKind, QueuedJob}, + utils::WarnAfterExt, worker::{to_raw_value, WORKER_GROUP}, DB, }; @@ -84,7 +91,49 @@ pub fn start_background_processor( JobKind::Dependencies | JobKind::FlowDependencies ); - handle_receive_completed_job( + let success = jc.success; + + let span = tracing::span!( + tracing::Level::INFO, + "job_postprocessing", + job_id = %jc.job.id, root_job = field::Empty, workspace_id = %jc.job.workspace_id, worker = %worker_name,tag = %jc.job.tag, + // hostname = %hostname, + language = field::Empty, + script_path = field::Empty, + flow_step_id = field::Empty, + parent_job = field::Empty, + otel.name = field::Empty + ); + let rj = if let Some(root_job) = jc.job.root_job { + root_job + } else { + jc.job.id + }; + windmill_common::otel_ee::set_span_parent(&span, &rj); + + if let Some(lg) = jc.job.language.as_ref() { + span.record("language", lg.as_str()); + } + if let Some(step_id) = jc.job.flow_step_id.as_ref() { + span.record( + "otel.name", + format!("job_postprocessing {}", step_id).as_str(), + ); + span.record("flow_step_id", step_id.as_str()); + } else { + span.record("otel.name", "job postprocessing"); + } + if let Some(parent_job) = jc.job.parent_job.as_ref() { + span.record("parent_job", parent_job.to_string().as_str()); + } + if let Some(script_path) = jc.job.script_path.as_ref() { + span.record("script_path", script_path.as_str()); + } + if let Some(root_job) = jc.job.root_job.as_ref() { + span.record("root_job", root_job.to_string().as_str()); + } + + let root_job = handle_receive_completed_job( jc, &base_internal_url, &db, @@ -95,8 +144,13 @@ pub fn start_background_processor( #[cfg(feature = "benchmark")] &mut bench, ) + .instrument(span) .await; + if let Some(root_job) = root_job { + windmill_common::otel_ee::add_root_flow_job_to_otlp(&root_job, success); + } + if is_init_script_and_failure { tracing::error!("init script errored, exiting"); killpill_tx.send(()).unwrap_or_default(); @@ -198,7 +252,11 @@ async fn send_job_completed( token, duration, }; - job_completed_tx.send(jc).await.expect("send job completed") + job_completed_tx + .send(jc) + .with_context(windmill_common::otel_ee::otel_ctx()) + .await + .expect("send job completed") } pub async fn process_result( @@ -270,6 +328,7 @@ pub async fn process_result( token, duration, ) + .with_context(windmill_common::otel_ee::otel_ctx()) .await; Ok(true) } @@ -314,6 +373,7 @@ pub async fn process_result( token, duration, ) + .with_context(windmill_common::otel_ee::otel_ctx()) .await; Ok(false) } @@ -329,7 +389,7 @@ pub async fn handle_receive_completed_job( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) { +) -> Option> { let token = jc.token.clone(); let workspace = jc.job.workspace_id.clone(); let client = AuthedClient { @@ -341,7 +401,7 @@ pub async fn handle_receive_completed_job( let job = jc.job.clone(); let mem_peak = jc.mem_peak.clone(); let canceled_by = jc.canceled_by.clone(); - if let Err(err) = process_completed_job( + match process_completed_job( jc, &client, db, @@ -354,26 +414,29 @@ pub async fn handle_receive_completed_job( ) .await { - handle_job_error( - db, - &client, - job.as_ref(), - mem_peak, - canceled_by, - err, - false, - same_worker_tx.clone(), - &worker_dir, - worker_name, - job_completed_tx, - #[cfg(feature = "benchmark")] - bench, - ) - .await; + Err(err) => { + handle_job_error( + db, + &client, + job.as_ref(), + mem_peak, + canceled_by, + err, + false, + same_worker_tx.clone(), + &worker_dir, + worker_name, + job_completed_tx, + #[cfg(feature = "benchmark")] + bench, + ) + .await; + None + } + Ok(r) => r, } } -#[tracing::instrument(name = "completed_job", level = "info", skip_all, fields(job_id = %job.id))] pub async fn process_completed_job( JobCompleted { job, result, mem_peak, success, cached_res_path, canceled_by, duration, .. }: JobCompleted, client: &AuthedClient, @@ -383,7 +446,7 @@ pub async fn process_completed_job( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) -> windmill_common::error::Result<()> { +) -> windmill_common::error::Result>> { if success { // println!("bef completed job{:?}", SystemTime::now()); if let Some(cached_path) = cached_res_path { @@ -413,8 +476,8 @@ pub async fn process_completed_job( if is_flow_step { if let Some(parent_job) = parent_job { - tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)"); - update_flow_status_after_job_completion( + // tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)"); + let r = update_flow_status_after_job_completion( db, client, parent_job, @@ -431,10 +494,12 @@ pub async fn process_completed_job( #[cfg(feature = "benchmark")] bench, ) + .warn_after_seconds(10) .await?; + add_time!(bench, "updated flow status END"); + return Ok(r); } } - add_time!(bench, "updated flow status END"); } else { let result = add_completed_job_error( db, @@ -452,7 +517,7 @@ pub async fn process_completed_job( if job.is_flow_step { if let Some(parent_job) = job.parent_job { tracing::error!(parent_flow = %parent_job, subflow = %job.id, "process completed job error, updating flow status"); - update_flow_status_after_job_completion( + let r = update_flow_status_after_job_completion( db, client, parent_job, @@ -469,11 +534,13 @@ pub async fn process_completed_job( #[cfg(feature = "benchmark")] bench, ) + .warn_after_seconds(10) .await?; + return Ok(r); } } } - Ok(()) + return Ok(None); } #[tracing::instrument(name = "job_error", level = "info", skip_all, fields(job_id = %job.id))] diff --git a/backend/windmill-worker/src/rust_executor.rs b/backend/windmill-worker/src/rust_executor.rs index 9391fd9a69..c6f258d6ac 100644 --- a/backend/windmill-worker/src/rust_executor.rs +++ b/backend/windmill-worker/src/rust_executor.rs @@ -209,6 +209,7 @@ pub async fn build_rust_crate( "TMP", std::env::var("TMP").unwrap_or_else(|_| "C:\\tmp".to_string()), ); + build_rust_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); } let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?; @@ -402,7 +403,10 @@ pub async fn handle_rust_job( .stderr(Stdio::piped()); #[cfg(windows)] - run_rust.env("SystemRoot", SYSTEM_ROOT.as_str()); + { + run_rust.env("SystemRoot", SYSTEM_ROOT.as_str()); + run_rust.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); + } start_child_process(run_rust, compiled_executable_name).await? }; diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 9f8d3e12fd..4e817a3927 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -419,6 +419,7 @@ pub async fn do_snowflake( worker_name, &job.workspace_id, &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await?; *mem_peak = (r.get().len() / 1000) as i32; diff --git a/backend/windmill-worker/src/windmill-client.js b/backend/windmill-worker/src/windmill-client.js index 64281fede9..3ae15e7453 100644 --- a/backend/windmill-worker/src/windmill-client.js +++ b/backend/windmill-worker/src/windmill-client.js @@ -558,6 +558,9 @@ var $QueuedJob = { "identity", "deploymentcallback", "singlescriptflow", + "flowscript", + "flownode", + "appscript", ], }, schedule_path: { @@ -697,6 +700,9 @@ var $CompletedJob = { "identity", "deploymentcallback", "singlescriptflow", + "flowscript", + "flownode", + "appscript", ], }, schedule_path: { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 0d64b30c5a..4af815cd76 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -6,9 +6,15 @@ * LICENSE-AGPL for a copy of the license. */ +// #[cfg(feature = "otel")] +// use opentelemetry::{global, KeyValue}; + + use windmill_common::{ + apps::AppScriptId, auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET}, scripts::PREVIEW_IS_TAR_CODEBASE_HASH, + utils::WarnAfterExt, worker::{ get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage, write_file, ROOT_CACHE_DIR, TMP_DIR, @@ -23,7 +29,7 @@ use const_format::concatcp; #[cfg(feature = "prometheus")] use prometheus::IntCounter; -use tracing::Instrument; +use tracing::{field, Instrument}; #[cfg(feature = "prometheus")] use windmill_common::METRICS_DEBUG_ENABLED; #[cfg(feature = "prometheus")] @@ -86,30 +92,12 @@ use tokio::{ use rand::Rng; use crate::{ - ansible_executor::handle_ansible_job, - bash_executor::{handle_bash_job, handle_powershell_job}, - bun_executor::handle_bun_job, - common::{ + ansible_executor::handle_ansible_job, bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{ build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, update_worker_ping_for_failed_init_script, OccupancyMetrics, - }, - deno_executor::handle_deno_job, - go_executor::handle_go_job, - graphql_executor::do_graphql, - handle_child::SLOW_LOGS, - handle_job_error, - job_logger::NO_LOGS_AT_ALL, - js_eval::{eval_fetch_timeout, transpile_ts}, - mysql_executor::do_mysql, - pg_executor::do_postgresql, - php_executor::handle_php_job, - python_executor::handle_python_job, - result_processor::{process_result, start_background_processor}, - rust_executor::handle_rust_job, - worker_flow::{handle_flow, update_flow_status_in_progress, Step}, - worker_lockfiles::{ + }, deno_executor::handle_deno_job, go_executor::handle_go_job, graphql_executor::do_graphql, handle_child::SLOW_LOGS, handle_job_error, job_logger::NO_LOGS_AT_ALL, js_eval::{eval_fetch_timeout, transpile_ts}, mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{process_result, start_background_processor}, rust_executor::handle_rust_job, worker_flow::{handle_flow, update_flow_status_in_progress, Step}, worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, - }, + } }; use backon::ConstantBuilder; @@ -159,6 +147,7 @@ pub async fn create_token_for_owner_in_bg( &email, &job_id, ) + .warn_after_seconds(5) .await .expect("could not create job token"); *locked = token; @@ -366,6 +355,7 @@ lazy_static::lazy_static! { pub static ref NSJAIL_PATH: String = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); pub static ref PATH_ENV: String = std::env::var("PATH").unwrap_or_else(|_| String::new()); pub static ref HOME_ENV: String = std::env::var("HOME").unwrap_or_else(|_| "/tmp".to_string()); + pub static ref NODE_PATH: Option = std::env::var("NODE_PATH").ok(); pub static ref TZ_ENV: String = std::env::var("TZ").unwrap_or_else(|_| String::new()); @@ -419,12 +409,12 @@ lazy_static::lazy_static! { .and_then(|x| x.parse().ok()) .unwrap_or(false); - } #[cfg(windows)] lazy_static::lazy_static! { pub static ref SYSTEM_ROOT: String = std::env::var("SystemRoot").unwrap_or_else(|_| "C:\\Windows".to_string()); + pub static ref USERPROFILE_ENV: String = std::env::var("USERPROFILE").unwrap_or_else(|_| "/tmp".to_string()); } //only matter if CLOUD_HOSTED @@ -716,7 +706,10 @@ fn add_outstanding_wait_time( }.in_current_span()); } -#[tracing::instrument(name = "worker", level = "info", skip_all, fields(worker = %worker_name, hostname = %hostname))] +// struct WorkerMtrics { +// job_ +// } + pub async fn run_worker( db: &Pool, hostname: &str, @@ -732,6 +725,7 @@ pub async fn run_worker( #[cfg(not(feature = "enterprise"))] if !*DISABLE_NSJAIL { tracing::warn!( + worker = %worker_name, hostname = %hostname, "NSJAIL to sandbox process in untrusted environments is an enterprise feature but allowed to be used for testing purposes" ); } @@ -739,10 +733,10 @@ pub async fn run_worker( let start_time = Instant::now(); let worker_dir = format!("{TMP_DIR}/{worker_name}"); - tracing::debug!(worker_dir = %worker_dir, "Creating worker dir"); + tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir"); if let Some(ref netrc) = *NETRC { - tracing::info!("Writing netrc at {}/.netrc", HOME_ENV.as_str()); + tracing::info!(worker = %worker_name, hostname = %hostname, "Writing netrc at {}/.netrc", HOME_ENV.as_str()); write_file(&HOME_ENV, ".netrc", netrc).expect("could not write netrc"); } @@ -957,6 +951,16 @@ pub async fn run_worker( None }; + + // let worker_resource = &[ + // KeyValue::new("hostname", hostname.to_string()), + // KeyValue::new("worker", worker_name.to_string()), + // ]; + // // Create a meter from the above MeterProvider. + // let meter = global::meter("windmill"); + // let counter = meter.u64_counter("jobs.execution").build(); + + let mut occupancy_metrics = OccupancyMetrics::new(start_time); let mut jobs_executed = 0; @@ -1012,6 +1016,7 @@ pub async fn run_worker( IS_READY.store(true, Ordering::Relaxed); tracing::info!( + worker = %worker_name, hostname = %hostname, "listening for jobs, WORKER_GROUP: {}, config: {:?}", *WORKER_GROUP, WORKER_CONFIG.read().await @@ -1047,7 +1052,7 @@ pub async fn run_worker( if i_worker == 1 { if let Err(e) = queue_init_bash_maybe(db, same_worker_tx.clone(), &worker_name).await { killpill_tx.send(()).unwrap_or_default(); - tracing::error!("Error queuing init bash script for worker {worker_name}: {e:#}"); + tracing::error!(worker = %worker_name, hostname = %hostname, "Error queuing init bash script for worker {worker_name}: {e:#}"); return; } } @@ -1076,11 +1081,12 @@ pub async fn run_worker( let mut last_suspend_first = Instant::now(); let mut killed_but_draining_same_worker_jobs = false; + let mut killpill_rx2 = killpill_rx.resubscribe(); loop { #[cfg(feature = "enterprise")] { if let Ok(_) = killpill_rx.try_recv() { - tracing::info!("killpill received on worker waiting for valid key"); + tracing::info!(worker = %worker_name, hostname = %hostname, "killpill received on worker waiting for valid key"); job_completed_tx .0 .send(SendResult::Kill) @@ -1092,6 +1098,7 @@ pub async fn run_worker( if !valid_key { tracing::error!( + worker = %worker_name, hostname = %hostname, "Invalid license key, workers require a valid license key, sleeping for 30s waiting for valid key to be set" ); tokio::time::sleep(Duration::from_secs(10)).await; @@ -1105,7 +1112,7 @@ pub async fn run_worker( #[cfg(feature = "prometheus")] if let Some(wk) = worker_busy.as_ref() { wk.set(0); - tracing::debug!("set worker busy to 0"); + tracing::debug!(worker = %worker_name, hostname = %hostname, "set worker busy to 0"); } occupancy_metrics.running_job_started_at = None; @@ -1117,7 +1124,7 @@ pub async fn run_worker( .try_into() .unwrap(), ); - tracing::debug!("set uptime metric"); + tracing::debug!(worker = %worker_name, hostname = %hostname, "set uptime metric"); } if last_ping.elapsed().as_secs() > NUM_SECS_PING { @@ -1161,15 +1168,19 @@ pub async fn run_worker( ) .notify(|err, dur| { tracing::error!( + worker = %worker_name, hostname = %hostname, "retrying updating worker ping in {dur:#?}, err: {err:#?}" ); }) .sleep(tokio::time::sleep) .await { - tracing::error!("failed to update worker ping, exiting: {}", e); + tracing::error!( + worker = %worker_name, hostname = %hostname, + "failed to update worker ping, exiting: {}", e); killpill_tx.send(()).unwrap_or_default(); } tracing::info!( + worker = %worker_name, hostname = %hostname, "ping update, memory: container={}MB, windmill={}MB", memory_usage.unwrap_or_default() / (1024 * 1024), wm_memory_usage.unwrap_or_default() / (1024 * 1024) @@ -1181,16 +1192,18 @@ pub async fn run_worker( if (jobs_executed as u32 + vacuum_shift) % VACUUM_PERIOD == 0 { let db2 = db.clone(); let current_span = tracing::Span::current(); + let worker_name = worker_name.clone(); + let hostname = hostname.to_string(); tokio::task::spawn( (async move { - tracing::info!("vacuuming queue and completed_job"); + tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue"); if let Err(e) = sqlx::query!("VACUUM (skip_locked) queue") .execute(&db2) .await { - tracing::error!("failed to vacuum queue: {}", e); + tracing::error!(worker = %worker_name, hostname = %hostname, "failed to vacuum queue: {}", e); } - tracing::info!("vacuumed queue and completed_job"); + tracing::info!(worker = %worker_name, hostname = %hostname, "vacuumed queue"); }) .instrument(current_span), ); @@ -1226,6 +1239,7 @@ pub async fn run_worker( if let Ok(same_worker_job) = same_worker_rx.try_recv() { same_worker_queue_size.fetch_sub(1, Ordering::SeqCst); tracing::debug!( + worker = %worker_name, hostname = %hostname, "received {} from same worker channel", same_worker_job.job_id ); @@ -1238,6 +1252,7 @@ pub async fn run_worker( .map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string())); if r.is_err() && !same_worker_job.recoverable { tracing::error!( + worker = %worker_name, hostname = %hostname, "failed to fetch same_worker job on a non recoverable job, exiting" ); job_completed_tx @@ -1251,7 +1266,7 @@ pub async fn run_worker( } } else if let Ok(_) = killpill_rx.try_recv() { if !killed_but_draining_same_worker_jobs { - tracing::info!("received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker); + tracing::info!(worker = %worker_name, hostname = %hostname, "received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker); killed_but_draining_same_worker_jobs = true; job_completed_tx .0 @@ -1262,10 +1277,10 @@ pub async fn run_worker( continue; } else if killed_but_draining_same_worker_jobs { if job_completed_processor_is_done.load(Ordering::SeqCst) { - tracing::info!("all running jobs have completed and all completed jobs have been fully processed, exiting"); + tracing::info!(worker = %worker_name, hostname = %hostname, "all running jobs have completed and all completed jobs have been fully processed, exiting"); break; } else { - tracing::info!("there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting"); + tracing::info!(worker = %worker_name, hostname = %hostname, "there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting"); tokio::time::sleep(Duration::from_millis(200)).await; continue; } @@ -1290,7 +1305,7 @@ pub async fn run_worker( if !agent_mode && duration_pull_s > 0.5 { let empty = job.as_ref().is_ok_and(|x| x.0.is_none()); - tracing::warn!("pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}"); + tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}"); #[cfg(feature = "prometheus")] if empty { if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() { @@ -1301,7 +1316,7 @@ pub async fn run_worker( } } else if !agent_mode && duration_pull_s > 0.1 { let empty = job.as_ref().is_ok_and(|x| x.0.is_none()); - tracing::warn!("pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}"); + tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}"); #[cfg(feature = "prometheus")] if empty { if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() { @@ -1355,7 +1370,7 @@ pub async fn run_worker( last_executed_job = None; jobs_executed += 1; - tracing::debug!("started handling of job {}", job.id); + tracing::debug!(worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id); if matches!(job.job_kind, JobKind::Script | JobKind::Preview) { if !dedicated_workers.is_empty() { @@ -1420,6 +1435,11 @@ pub async fn run_worker( ) .await; + // counter.add( + // 1, + // worker_resource + // ); + #[cfg(feature = "prometheus")] let _timer = register_metric( &WORKER_EXECUTION_DURATION, @@ -1507,6 +1527,40 @@ pub async fn run_worker( let PulledJob { job, raw_code, raw_lock, raw_flow } = job; let arc_job = Arc::new(job); add_time!(bench, "handle_queued_job START"); + + + let span = tracing::span!(tracing::Level::INFO, "job", + job_id = %arc_job.id, root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, hostname = %hostname, tag = %arc_job.tag, + language = field::Empty, + script_path = field::Empty, flow_step_id = field::Empty, parent_job = field::Empty, + otel.name = field::Empty); + let rj = if let Some(root_job) = arc_job.root_job { + root_job + } else { + arc_job.id + }; + if let Some(lg) = arc_job.language.as_ref() { + span.record("language", lg.as_str()); + } + if let Some(step_id) = arc_job.flow_step_id.as_ref() { + span.record("otel.name", format!("job {}", step_id).as_str()); + span.record("flow_step_id", step_id.as_str()); + } else { + span.record("otel.name", "job"); + } + if let Some(parent_job) = arc_job.parent_job.as_ref() { + span.record("parent_job", parent_job.to_string().as_str()); + } + if let Some(script_path) = arc_job.script_path.as_ref() { + span.record("script_path", script_path.as_str()); + } + if let Some(root_job) = arc_job.root_job.as_ref() { + span.record("root_job", root_job.to_string().as_str()); + } + + windmill_common::otel_ee::set_span_parent(&span, &rj); + // span.context().span().add_event_with_timestamp("job created".to_string(), arc_job.created_at.into(), vec![]); + match handle_queued_job( arc_job.clone(), raw_code, @@ -1522,9 +1576,11 @@ pub async fn run_worker( base_internal_url, job_completed_tx.clone(), &mut occupancy_metrics, + &mut killpill_rx2, #[cfg(feature = "benchmark")] &mut bench, ) + .instrument(span) .await { Err(err) => { @@ -1564,6 +1620,8 @@ pub async fn run_worker( _ => {} } + + #[cfg(feature = "prometheus")] if let Some(duration) = _timer.map(|x| x.stop_and_record()) { register_metric( @@ -1603,7 +1661,7 @@ pub async fn run_worker( if let Some(secs) = *EXIT_AFTER_NO_JOB_FOR_SECS { if let Some(lj) = last_executed_job { if lj.elapsed().as_secs() > secs { - tracing::info!("no job for {} seconds, exiting", secs); + tracing::info!(worker = %worker_name, hostname = %hostname, "no job for {} seconds, exiting", secs); break; } } else { @@ -1634,12 +1692,12 @@ pub async fn run_worker( }); } Err(err) => { - tracing::error!("Failed to pull jobs: {}", err); + tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err); } }; } - tracing::info!("worker {} exiting", worker_name); + tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exiting", worker_name); #[cfg(feature = "benchmark")] { @@ -1654,22 +1712,24 @@ pub async fn run_worker( if has_dedicated_workers { for handle in dedicated_handles { if let Err(e) = handle.await { - tracing::error!("error in dedicated worker waiting for it to end: {:?}", e) + tracing::error!(worker = %worker_name, hostname = %hostname, "error in dedicated worker waiting for it to end: {:?}", e) } } - tracing::info!("all dedicated workers have exited"); + tracing::info!(worker = %worker_name, hostname = %hostname, "all dedicated workers have exited"); } drop(job_completed_tx); - tracing::info!("waiting for job_completed_processor to finish processing remaining jobs"); + tracing::info!(worker = %worker_name, hostname = %hostname, "waiting for job_completed_processor to finish processing remaining jobs"); if let Err(e) = send_result.await { tracing::error!("error in awaiting send_result process: {e:?}") } - tracing::info!("worker {} exited", worker_name); - tracing::info!("number of jobs executed: {}", jobs_executed); + tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name); + tracing::info!(worker = %worker_name, hostname = %hostname, "number of jobs executed: {}", jobs_executed); } + + async fn queue_init_bash_maybe<'c>( db: &Pool, same_worker_tx: SameWorkerSender, @@ -1794,7 +1854,6 @@ pub struct PreviousResult<'a> { pub previous_result: Option<&'a RawValue>, } -#[tracing::instrument(name = "job", level = "info", skip_all, fields(job_id = %job.id))] async fn handle_queued_job( job: Arc, raw_code: Option, @@ -1810,8 +1869,12 @@ async fn handle_queued_job( base_internal_url: &str, job_completed_tx: JobCompletedSender, occupancy_metrics: &mut OccupancyMetrics, + killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { + // Extract the active span from the context + + if job.canceled { return Err(Error::JsonErr(canceled_job_to_result(&job))); } @@ -1823,7 +1886,9 @@ async fn handle_queued_job( if job.parent_job.is_none() && job.created_by.starts_with("email-") { let daily_count = sqlx::query!( "SELECT value FROM metrics WHERE id = 'email_trigger_usage' AND created_at > NOW() - INTERVAL '1 day' ORDER BY created_at DESC LIMIT 1" - ).fetch_optional(db).await?.map(|x| serde_json::from_value::(x.value).unwrap_or(1)); + ).fetch_optional(db) + .warn_after_seconds(5) + .await?.map(|x| serde_json::from_value::(x.value).unwrap_or(1)); if let Some(count) = daily_count { if count >= 100 { @@ -1836,6 +1901,7 @@ async fn handle_queued_job( serde_json::json!(count + 1) ) .execute(db) + .warn_after_seconds(5) .await?; } } else { @@ -1843,6 +1909,7 @@ async fn handle_queued_job( "INSERT INTO metrics (id, value) VALUES ('email_trigger_usage', to_jsonb(1))" ) .execute(db) + .warn_after_seconds(5) .await?; } } @@ -1855,6 +1922,7 @@ async fn handle_queued_job( .ok_or_else(|| Error::InternalErr(format!("expected parent job")))?, job.id, ) + .warn_after_seconds(5) .await?; Some(r) @@ -1867,6 +1935,7 @@ async fn handle_queued_job( &job.workspace_id ) .execute(db) + .warn_after_seconds(5) .await { tracing::error!("Could not update parent job started_at flow_status: {}", e); } @@ -1883,6 +1952,7 @@ async fn handle_queued_job( job.workspace_id ) .fetch_one(db) + .warn_after_seconds(5) .await .map(|record| (record.raw_code, record.raw_lock, record.raw_flow)) .unwrap_or_default(), @@ -1923,6 +1993,7 @@ async fn handle_queued_job( &job.parent_job.unwrap() ) .fetch_one(db) + .warn_after_seconds(5) .await .map_err(|e| { Error::InternalErr(format!( @@ -1957,6 +2028,7 @@ async fn handle_queued_job( &job.workspace_id, &cached_res_path, ) + .warn_after_seconds(5) .await; if let Some(cached_resource_value) = cached_resource_value_maybe { { @@ -1995,6 +2067,7 @@ async fn handle_queued_job( worker_dir, job_completed_tx.0.clone(), ) + .warn_after_seconds(10) .await?; Ok(true) } else { @@ -2107,6 +2180,7 @@ async fn handle_queued_job( &mut column_order, &mut new_args, occupancy_metrics, + killpill_rx, ) .await; occupancy_metrics.total_duration_of_running_jobs += @@ -2145,6 +2219,7 @@ async fn handle_queued_job( } } + pub fn build_envs( envs: Option>, ) -> windmill_common::error::Result> { @@ -2250,6 +2325,8 @@ async fn handle_code_execution_job( column_order: &mut Option>, new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, + killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, + ) -> error::Result> { let ContentReqLangEnvs { content: inner_content, @@ -2285,9 +2362,25 @@ async fn handle_code_execution_job( .await? } JobKind::FlowScript => { - let (lockfile, content) = cache::flow::fetch_script(db, FlowNodeId( - job.script_hash.unwrap_or(ScriptHash(0)).0 - )).await?; + let (lockfile, content) = cache::flow::fetch_script( + db, + FlowNodeId(job.script_hash.unwrap_or(ScriptHash(0)).0), + ) + .await?; + ContentReqLangEnvs { + content, + lockfile, + language: job.language.to_owned(), + envs: None, + codebase: None, + } + } + JobKind::AppScript => { + let (lockfile, content) = cache::app::fetch_script( + db, + AppScriptId(job.script_hash.unwrap_or(ScriptHash(0)).0), + ) + .await?; ContentReqLangEnvs { content, lockfile, @@ -2577,6 +2670,7 @@ mount {{ worker_name, envs, occupancy_metrics, + killpill_rx, ) .await } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 33b17db790..b5770166c5 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -41,6 +41,7 @@ use windmill_common::jobs::{ script_hash_to_tag_and_limits, script_path_to_payload, BranchResults, JobPayload, QueuedJob, RawCode, ENTRYPOINT_OVERRIDE, }; +use windmill_common::utils::WarnAfterExt; use windmill_common::worker::to_raw_value; use windmill_common::{ error::{self, to_anyhow, Error}, @@ -76,45 +77,34 @@ pub async fn update_flow_status_after_job_completion( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) -> error::Result<()> { +) -> error::Result>> { // this is manual tailrecursion because async_recursion blows up the stack - // todo!(); potentially_crash_for_testing(); - let mut rec = update_flow_status_after_job_completion_internal( - db, - client, + let mut rec = RecUpdateFlowStatusAfterJobCompletion { flow, - job_id_for_status, - w_id, + job_id_for_status: job_id_for_status.clone(), success, result, - unrecoverable, - same_worker_tx.clone(), - worker_dir, stop_early_override, - false, - worker_name, - job_completed_tx.clone(), - #[cfg(feature = "benchmark")] - bench, - ) - .await?; - while let Some(nrec) = rec { + skip_error_handler: false, + }; + let mut unrecoverable = unrecoverable; + loop { potentially_crash_for_testing(); - rec = match update_flow_status_after_job_completion_internal( + let nrec = match update_flow_status_after_job_completion_internal( db, client, - nrec.flow, - &nrec.job_id_for_status, + rec.flow, + &rec.job_id_for_status, w_id, - nrec.success, - nrec.result, - false, + rec.success, + rec.result, + unrecoverable, same_worker_tx.clone(), worker_dir, - nrec.stop_early_override, - nrec.skip_error_handler, + rec.stop_early_override, + rec.skip_error_handler, worker_name, job_completed_tx.clone(), #[cfg(feature = "benchmark")] @@ -124,12 +114,12 @@ pub async fn update_flow_status_after_job_completion( { Ok(j) => j, Err(e) => { - tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", nrec.flow,&nrec.job_id_for_status); + tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", rec.flow, &rec.job_id_for_status); update_flow_status_after_job_completion_internal( db, client, - nrec.flow, - &nrec.job_id_for_status, + rec.flow, + &rec.job_id_for_status, w_id, false, Arc::new(to_raw_value(&Json(&WrappedError { @@ -138,8 +128,8 @@ pub async fn update_flow_status_after_job_completion( true, same_worker_tx.clone(), worker_dir, - nrec.stop_early_override, - nrec.skip_error_handler, + rec.stop_early_override, + rec.skip_error_handler, worker_name, job_completed_tx.clone(), #[cfg(feature = "benchmark")] @@ -147,9 +137,33 @@ pub async fn update_flow_status_after_job_completion( ) .await? } + }; + unrecoverable = false; + match nrec { + UpdateFlowStatusAfterJobCompletion::Done(job) => { + add_time!(bench, "update flow status internal END"); + return Ok(Some(job)); + } + UpdateFlowStatusAfterJobCompletion::Rec(nrec) => { + rec = nrec; + }, + UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch => { + add_time!(bench, "update flow status internal END"); + return Ok(None); + }, + UpdateFlowStatusAfterJobCompletion::NotDone => { + add_time!(bench, "update flow status internal END"); + return Ok(None); + } } } - Ok(()) +} + +pub enum UpdateFlowStatusAfterJobCompletion { + Rec(RecUpdateFlowStatusAfterJobCompletion), + Done(Arc), + NotDone, + NonLastParallelBranch, } pub struct RecUpdateFlowStatusAfterJobCompletion { flow: uuid::Uuid, @@ -187,7 +201,7 @@ pub async fn update_flow_status_after_job_completion_internal( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) -> error::Result> { +) -> error::Result { add_time!(bench, "update flow status internal START"); let ( should_continue_flow, @@ -602,7 +616,7 @@ pub async fn update_flow_status_after_job_completion_internal( ); } add_time!(bench, "non final parallel flow finished"); - return Ok(None); + return Ok(UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch); } } FlowStatusModule::InProgress { @@ -1109,6 +1123,7 @@ pub async fn update_flow_status_after_job_completion_internal( worker_dir, job_completed_tx, ) + .warn_after_seconds(10) .await { Err(err) => { @@ -1146,7 +1161,7 @@ pub async fn update_flow_status_after_job_completion_internal( if let Some(parent_job) = flow_job.parent_job { tracing::info!(subflow_id = %flow_job.id, parent_id = %parent_job, "subflow is finished, updating parent flow status"); - return Ok(Some(RecUpdateFlowStatusAfterJobCompletion { + return Ok(UpdateFlowStatusAfterJobCompletion::Rec(RecUpdateFlowStatusAfterJobCompletion { flow: parent_job, job_id_for_status: flow, success: success && !is_failure_step, @@ -1160,9 +1175,9 @@ pub async fn update_flow_status_after_job_completion_internal( })); } } - Ok(None) + Ok(UpdateFlowStatusAfterJobCompletion::Done(flow_job)) } else { - Ok(None) + Ok(UpdateFlowStatusAfterJobCompletion::NotDone) } } @@ -1290,7 +1305,7 @@ async fn compute_skip_branchall_failure<'c>( // ))) // } -fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u16, Duration)> { +fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u32, Duration)> { (status.fail_count <= MAX_RETRY_ATTEMPTS) .then(|| &retry) .and_then(|retry| retry.interval(status.fail_count, false)) @@ -1510,7 +1525,7 @@ pub async fn handle_flow( let schedule_path = flow_job.schedule_path.as_ref().unwrap(); let schedule = - get_schedule_opt(&mut tx, &flow_job.workspace_id, schedule_path).await?; + get_schedule_opt(&mut tx, &flow_job.workspace_id, schedule_path).warn_after_seconds(5).await?; tx.commit().await?; @@ -1522,6 +1537,7 @@ pub async fn handle_flow( flow_job.script_path.as_ref().unwrap(), &flow_job.workspace_id, ) + .warn_after_seconds(5) .await { match err { @@ -1549,6 +1565,7 @@ pub async fn handle_flow( worker_dir, job_completed_tx, ) + .warn_after_seconds(10) .await?; Ok(()) } @@ -2133,6 +2150,7 @@ async fn push_next_flow_job( .bind(status.step) .bind(json!(status.retry.failed_jobs)) .execute(db) + .warn_after_seconds(2) .await .context("update flow retry")?; @@ -2558,6 +2576,7 @@ async fn push_next_flow_job( new_job_priority_override, job_perms.as_ref(), ) + .warn_after_seconds(2) .await?; tracing::debug!(id = %flow_job.id, root_id = %job_root, "pushed next flow job: {uuid}"); @@ -2770,7 +2789,7 @@ async fn push_next_flow_job( .execute(&mut *tx) .await?; - tx.commit().await?; + tx.commit().warn_after_seconds(3).await?; tracing::info!(id = %flow_job.id, root_id = %job_root, "all next flow jobs pushed: {uuids:?}"); if continue_on_same_worker { diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index f684d6aa0f..787f821008 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -4,6 +4,7 @@ use std::path::{Component, Path, PathBuf}; use async_recursion::async_recursion; use serde_json::value::RawValue; use serde_json::{json, Value}; +use sha2::Digest; use sqlx::types::Json; use uuid::Uuid; use windmill_common::error::Error; @@ -14,6 +15,7 @@ use windmill_common::jobs::JobPayload; use windmill_common::scripts::ScriptHash; use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, PythonAnnotations}; use windmill_common::{ + apps::AppScriptId, error::{self, to_anyhow}, flows::{add_virtual_items_if_necessary, FlowValue}, jobs::QueuedJob, @@ -606,9 +608,8 @@ pub async fn handle_flow_dependency_job( occupancy_metrics, ) .await?; - let new_flow_value = sqlx::types::Json( - serde_json::value::to_raw_value(&flow).map_err(to_anyhow)? - ); + let new_flow_value = + sqlx::types::Json(serde_json::value::to_raw_value(&flow).map_err(to_anyhow)?); // Re-check cancelation to ensure we don't accidentially override a flow. if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id) @@ -648,11 +649,20 @@ pub async fn handle_flow_dependency_job( // Compute a lite version of the flow value (`RawScript` => `FlowScript`). let mut value_lite = flow.clone(); - tx = reduce(tx, &mut value_lite.modules, &job_path, &job.workspace_id, flow.failure_module.as_ref(), flow.same_worker).await?; + tx = reduce_flow( + tx, + &mut value_lite.modules, + &job_path, + &job.workspace_id, + flow.failure_module.as_ref(), + flow.same_worker, + ) + .await?; sqlx::query!( "INSERT INTO flow_version_lite (id, value) VALUES ($1, $2) ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value", - version, sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json>, + version, + sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json>, ) .execute(db) .await?; @@ -829,8 +839,12 @@ async fn lock_modules<'c>( occupancy_metrics, )) .await?; - e.value = - FlowModuleValue::WhileloopFlow { modules: nmodules, modules_node, skip_failures }.into() + e.value = FlowModuleValue::WhileloopFlow { + modules: nmodules, + modules_node, + skip_failures, + } + .into() } FlowModuleValue::BranchOne { branches, default, default_node } => { let mut nbranches = vec![]; @@ -878,8 +892,12 @@ async fn lock_modules<'c>( occupancy_metrics, )) .await?; - e.value = FlowModuleValue::BranchOne { branches: nbranches, default: ndefault, default_node } - .into(); + e.value = FlowModuleValue::BranchOne { + branches: nbranches, + default: ndefault, + default_node, + } + .into(); } _ => (), }; @@ -1009,44 +1027,68 @@ async fn insert_flow_node<'c>( flow: Option<&Json>>, ) -> Result<(sqlx::Transaction<'c, sqlx::Postgres>, FlowNodeId)> { let hash = { - use std::hash::{DefaultHasher, Hasher, Hash}; - - let mut hasher = DefaultHasher::new(); - code.hash(&mut hasher); - lock.hash(&mut hasher); - flow.inspect(|flow| flow.get().hash(&mut hasher)); - hasher.finish() as i64 + let mut hasher = sha2::Sha256::new(); + hasher.update(code.unwrap_or(&Default::default())); + hasher.update(lock.unwrap_or(&Default::default())); + hasher.update(flow.unwrap_or(&Default::default()).get()); + format!("{:x}", hasher.finalize()) }; // Insert the flow node if it doesn't exist. let id = sqlx::query_scalar!( r#" - WITH existing AS ( - SELECT id FROM flow_node - WHERE hash = $1 AND path = $2 AND workspace_id = $3 - AND (code IS NOT DISTINCT FROM $4) - AND (lock IS NOT DISTINCT FROM $5) - AND (flow IS NOT DISTINCT FROM $6) - LIMIT 1 - ), - inserted AS ( - INSERT INTO flow_node (hash, path, workspace_id, code, lock, flow) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT DO NOTHING - RETURNING id - ) - SELECT id FROM existing - UNION ALL - SELECT id FROM inserted + INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id + RETURNING id "#, - hash, path, workspace_id, code, lock, flow as Option<&Json>> + path, + workspace_id, + hash, + lock, + code, + flow as Option<&Json>> ) .fetch_one(&mut *tx) - .await? - .ok_or(error::Error::InternalErr("Failed to cache".to_string()))?; + .await?; Ok((tx, FlowNodeId(id))) } +async fn insert_app_script( + db: &sqlx::Pool, + app: i64, + code: String, + lock: Option, +) -> Result { + let code_sha256 = format!("{:x}", sha2::Sha256::digest(&code)); + let hash = { + let mut hasher = sha2::Sha256::new(); + hasher.update(app.to_le_bytes()); + hasher.update(&code_sha256); + hasher.update(lock.as_ref().unwrap_or(&Default::default())); + format!("{:x}", hasher.finalize()) + }; + + // Insert the app script if it doesn't exist. + sqlx::query_scalar!( + r#" + INSERT INTO app_script (app, hash, lock, code, code_sha256) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id + RETURNING id + "#, + app, + hash, + lock, + code, + code_sha256 + ) + .fetch_one(db) + .await + .map(AppScriptId) + .map_err(Into::into) +} + async fn insert_flow_modules<'c>( mut tx: sqlx::Transaction<'c, sqlx::Postgres>, path: &str, @@ -1056,7 +1098,15 @@ async fn insert_flow_modules<'c>( modules: &mut Vec, modules_node: &mut Option, ) -> Result> { - tx = Box::pin(reduce(tx, modules, path, workspace_id, failure_module, same_worker)).await?; + tx = Box::pin(reduce_flow( + tx, + modules, + path, + workspace_id, + failure_module, + same_worker, + )) + .await?; add_virtual_items_if_necessary(modules); if modules.is_empty() || crate::worker_flow::is_simple_modules(modules, failure_module) { return Ok(tx); @@ -1073,14 +1123,14 @@ async fn insert_flow_modules<'c>( failure_module: failure_module.cloned(), same_worker, ..Default::default() - }))) + }))), ) .await?; *modules_node = Some(id); Ok(tx) } -async fn reduce<'c>( +async fn reduce_flow<'c>( mut tx: sqlx::Transaction<'c, sqlx::Postgres>, modules: &mut Vec, path: &str, @@ -1090,8 +1140,13 @@ async fn reduce<'c>( ) -> Result> { use FlowModuleValue::*; for module in &mut *modules { - let mut val = serde_json::from_str::(module.value.get()) - .map_err(|err| Error::InternalErr(format!("reduce: Failed to parse flow module value: {}", err)))?; + let mut val = + serde_json::from_str::(module.value.get()).map_err(|err| { + Error::InternalErr(format!( + "reduce_flow: Failed to parse flow module value: {}", + err + )) + })?; match &mut val { RawScript { .. } => { // In order to avoid an unnecessary `.clone()` of `val`, take ownership of it's content @@ -1107,9 +1162,14 @@ async fn reduce<'c>( concurrency_time_window_s, is_trigger, .. - } = std::mem::replace(&mut val, Identity) else { unreachable!() }; + } = std::mem::replace(&mut val, Identity) + else { + unreachable!() + }; let id; - (tx, id) = insert_flow_node(tx, path, workspace_id, Some(&content), lock.as_ref(), None).await?; + (tx, id) = + insert_flow_node(tx, path, workspace_id, Some(&content), lock.as_ref(), None) + .await?; val = FlowScript { input_transforms, id, @@ -1120,32 +1180,56 @@ async fn reduce<'c>( concurrency_time_window_s, is_trigger, }; - }, + } ForloopFlow { modules, modules_node, .. } - | WhileloopFlow { modules, modules_node, .. } => { + | WhileloopFlow { modules, modules_node, .. } => { tx = insert_flow_modules( - tx, path, workspace_id, failure_module, same_worker, - modules, modules_node - ).await?; + tx, + path, + workspace_id, + failure_module, + same_worker, + modules, + modules_node, + ) + .await?; } BranchOne { branches, default, default_node, .. } => { for branch in branches.iter_mut() { tx = insert_flow_modules( - tx, path, workspace_id, failure_module, same_worker, - &mut branch.modules, &mut branch.modules_node - ).await?; + tx, + path, + workspace_id, + failure_module, + same_worker, + &mut branch.modules, + &mut branch.modules_node, + ) + .await?; } tx = insert_flow_modules( - tx, path, workspace_id, failure_module, same_worker, - default, default_node - ).await?; + tx, + path, + workspace_id, + failure_module, + same_worker, + default, + default_node, + ) + .await?; } BranchAll { branches, .. } => { for branch in branches.iter_mut() { tx = insert_flow_modules( - tx, path, workspace_id, failure_module, same_worker, - &mut branch.modules, &mut branch.modules_node - ).await?; + tx, + path, + workspace_id, + failure_module, + same_worker, + &mut branch.modules, + &mut branch.modules_node, + ) + .await?; } } _ => {} @@ -1155,6 +1239,41 @@ async fn reduce<'c>( Ok(tx) } +async fn reduce_app(db: &sqlx::Pool, value: &mut Value, app: i64) -> Result<()> { + match value { + Value::Object(object) => { + if let Some(Value::Object(script)) = object.get_mut("inlineScript") { + // replace `content` with an empty string: + let Some(Value::String(code)) = script.get_mut("content").map(std::mem::take) + else { + return Err(error::Error::InternalErr( + "Missing `content` in inlineScript".to_string(), + )); + }; + // remove `lock`: + let lock = script.remove("lock").and_then(|x| match x { + Value::String(s) => Some(s), + _ => None, + }); + let id = insert_app_script(db, app, code, lock).await?; + // insert the `id` into the `script` object: + script.insert("id".to_string(), json!(id.0)); + } else { + for (_, value) in object { + Box::pin(reduce_app(db, value, app)).await?; + } + } + } + Value::Array(array) => { + for value in array { + Box::pin(reduce_app(db, value, app)).await?; + } + } + _ => {} + } + Ok(()) +} + fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool { if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative { let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content); @@ -1340,11 +1459,12 @@ pub async fn handle_app_dependency_job( .clone() .ok_or_else(|| Error::InternalErr("App Dependency requires script hash".to_owned()))? .0; - let value = sqlx::query_scalar!("SELECT value FROM app_version WHERE id = $1", id) + let record = sqlx::query!("SELECT app_id, value FROM app_version WHERE id = $1", id) .fetch_optional(db) - .await?; + .await? + .map(|record| (record.app_id, record.value)); - if let Some(value) = value { + if let Some((app_id, value)) = record { let value = lock_modules_app( value, job, @@ -1361,6 +1481,21 @@ pub async fn handle_app_dependency_job( ) .await?; + // Compute a lite version of the app value (w/ `inlineScript.{lock,code}`). + let mut value_lite = value.clone(); + reduce_app(db, &mut value_lite, app_id).await?; + if let Value::Object(object) = &mut value_lite { + object.insert("version".to_string(), json!(id)); + } + sqlx::query!( + "INSERT INTO app_version_lite (id, value) VALUES ($1, $2) + ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value", + id, + sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json>, + ) + .execute(db) + .await?; + // Re-check cancelation to ensure we don't accidentially override an app. if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id) .fetch_optional(db) diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index da88c5d0aa..2848c702dc 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.434.2"; +export const VERSION = "v1.435.2"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index bf465fc315..1bad532ef9 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.434.2"; +export const VERSION = "1.435.2"; const command = new Command() .name("wmill") diff --git a/docker-compose.yml b/docker-compose.yml index d45e589808..13d1c3ac98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -129,12 +129,6 @@ services: - PORT=8001 - DATABASE_URL=${DATABASE_URL} - MODE=indexer - - TANTIVY_MAX_INDEXED_JOB_LOG_SIZE__MB=1 # job logs bigger than this will be truncated before indexing - - TANTIVY_S3_BACKUP_PERIOD__S=3600 # how often to backup the index into object storage - - TANTIVY_INDEX_WRITER_MEMORY_BUDGET__MB=100 # higher budget for higher indexing throughput - - TANTIVY_REFRESH_INDEX_PERIOD__S=300 #how often to start indexing new jobs - - TANTIVY_DOC_COMMIT_MAX_BATCH_SIZE=100000 #how many documents to batch in one commit - - TANTIVY_SHOW_MEMORY_EVERY=10000 #log memory usage and progress every so many documents indexed depends_on: db: condition: service_healthy diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index a1b5daca43..35b8b8ba6a 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 94cc160cae..44dd185d25 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 237bcb81bd..300cb2649c 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.434.2", + "version": "1.435.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.434.2", + "version": "1.435.2", "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.32.1", @@ -87,10 +87,10 @@ "@hey-api/openapi-ts": "^0.43.0", "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", - "@sveltejs/adapter-static": "^3.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/adapter-static": "^3.0.6", + "@sveltejs/kit": "^2.9.0", "@sveltejs/package": "^2.2.2", - "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@sveltejs/vite-plugin-svelte": "^3.1.2", "@tailwindcss/forms": "^0.5.3", "@tailwindcss/typography": "^0.5.8", "@types/d3": "^7.4.0", @@ -130,7 +130,7 @@ "tailwindcss": "^3.4.1", "tslib": "^2.6.1", "typescript": "^5.1.3", - "vite": "^5", + "vite": "^5.4.11", "vite-plugin-circular-dependency": "^0.2.1", "vite-plugin-mkcert": "^1.17.5", "yootils": "^0.3.1" @@ -3584,9 +3584,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==" + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.20", @@ -3897,10 +3897,11 @@ } }, "node_modules/@polka/url": { - "version": "1.0.0-next.24", - "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.24.tgz", - "integrity": "sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ==", - "dev": true + "version": "1.0.0-next.28", + "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.28.tgz", + "integrity": "sha512-8LduaNlMZGwdZ6qWrKlfa+2M4gahzFkprZiAt2TF8uS0qQgBizKXpXURqvTJ4WtmupWxaLqjRb2UCTe72mu+Aw==", + "dev": true, + "license": "MIT" }, "node_modules/@popperjs/core": { "version": "2.11.8", @@ -4214,32 +4215,33 @@ } }, "node_modules/@sveltejs/adapter-static": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.1.tgz", - "integrity": "sha512-6lMvf7xYEJ+oGeR5L8DFJJrowkefTK6ZgA4JiMqoClMkKq0s6yvsd3FZfCFvX1fQ0tpCD7fkuRVHsnUVgsHyNg==", + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.6.tgz", + "integrity": "sha512-MGJcesnJWj7FxDcB/GbrdYD3q24Uk0PIL4QIX149ku+hlJuj//nxUbb0HxUTpjkecWfHjVveSUnUaQWnPRXlpg==", "dev": true, "peerDependencies": { "@sveltejs/kit": "^2.0.0" } }, "node_modules/@sveltejs/kit": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.5.0.tgz", - "integrity": "sha512-1uyXvzC2Lu1FZa30T4y5jUAC21R309ZMRG0TPt+PPPbNUoDpy8zSmSNVWYaBWxYDqLGQ5oPNWvjvvF2IjJ1jmA==", + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.9.0.tgz", + "integrity": "sha512-W3E7ed3ChB6kPqRs2H7tcHp+Z7oiTFC6m+lLyAQQuyXeqw6LdNuuwEUla+5VM0OGgqQD+cYD6+7Xq80vVm17Vg==", "dev": true, "hasInstallScript": true, + "license": "MIT", "dependencies": { "@types/cookie": "^0.6.0", "cookie": "^0.6.0", - "devalue": "^4.3.2", - "esm-env": "^1.0.0", - "import-meta-resolve": "^4.0.0", + "devalue": "^5.1.0", + "esm-env": "^1.2.1", + "import-meta-resolve": "^4.1.0", "kleur": "^4.1.5", "magic-string": "^0.30.5", "mrmime": "^2.0.0", "sade": "^1.8.1", "set-cookie-parser": "^2.6.0", - "sirv": "^2.0.4", + "sirv": "^3.0.0", "tiny-glob": "^0.2.9" }, "bin": { @@ -4249,9 +4251,9 @@ "node": ">=18.13" }, "peerDependencies": { - "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0", "svelte": "^4.0.0 || ^5.0.0-next.0", - "vite": "^5.0.3" + "vite": "^5.0.3 || ^6.0.0" } }, "node_modules/@sveltejs/package": { @@ -4277,17 +4279,17 @@ } }, "node_modules/@sveltejs/vite-plugin-svelte": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-3.0.2.tgz", - "integrity": "sha512-MpmF/cju2HqUls50WyTHQBZUV3ovV/Uk8k66AN2gwHogNAG8wnW8xtZDhzNBsFJJuvmq1qnzA5kE7YfMJNFv2Q==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-3.1.2.tgz", + "integrity": "sha512-Txsm1tJvtiYeLUVRNqxZGKR/mI+CzuIQuc2gn+YCs9rMTowpNZ2Nqt53JdL8KF9bLhAf2ruR/dr9eZCwdTriRA==", "dev": true, "dependencies": { - "@sveltejs/vite-plugin-svelte-inspector": "^2.0.0", + "@sveltejs/vite-plugin-svelte-inspector": "^2.1.0", "debug": "^4.3.4", "deepmerge": "^4.3.1", "kleur": "^4.1.5", - "magic-string": "^0.30.5", - "svelte-hmr": "^0.15.3", + "magic-string": "^0.30.10", + "svelte-hmr": "^0.16.0", "vitefu": "^0.2.5" }, "engines": { @@ -4299,9 +4301,9 @@ } }, "node_modules/@sveltejs/vite-plugin-svelte-inspector": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-2.0.0.tgz", - "integrity": "sha512-gjr9ZFg1BSlIpfZ4PRewigrvYmHWbDrq2uvvPB1AmTWKuM+dI1JXQSUu2pIrYLb/QncyiIGkFDFKTwJ0XqQZZg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-2.1.0.tgz", + "integrity": "sha512-9QX28IymvBlSCqsCll5t0kQVxipsfhFFL+L2t3nTWfXnddYwxBuAEtTtlaVQpRz9c37BhJjltSeY4AJSC03SSg==", "dev": true, "dependencies": { "debug": "^4.3.4" @@ -6434,10 +6436,11 @@ } }, "node_modules/devalue": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/devalue/-/devalue-4.3.2.tgz", - "integrity": "sha512-KqFl6pOgOW+Y6wJgu80rHpo2/3H07vr8ntR9rkkFIRETewbf5GaYYcakYfiKz89K+sLsuPkQIZaXDMjUObZwWg==", - "dev": true + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.1.1.tgz", + "integrity": "sha512-maua5KUiapvEwiEAe+XnlZ3Rh0GD+qI1J/nb9vrJc3muPXvcF/8gXYTWF76+5DAqHyDUtOIImEuo0YKE9mshVw==", + "dev": true, + "license": "MIT" }, "node_modules/devlop": { "version": "1.1.0", @@ -6990,9 +6993,10 @@ } }, "node_modules/esm-env": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.0.0.tgz", - "integrity": "sha512-Cf6VksWPsTuW01vU9Mk/3vRue91Zevka5SjyNf3nEpokFRuqt/KjUQoGAwq9qMmhpLTHmXzSIrFRw8zxWzmFBA==" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.1.tgz", + "integrity": "sha512-U9JedYYjCnadUlXk7e1Kr+aENQhtUaoaV9+gZm1T8LC/YBAPJx3NSPIAurFOC0U5vrdSevnUJS2/wUVxGwPhng==", + "license": "MIT" }, "node_modules/esm-env-robust": { "version": "0.0.3", @@ -7932,10 +7936,11 @@ } }, "node_modules/import-meta-resolve": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.0.0.tgz", - "integrity": "sha512-okYUR7ZQPH+efeuMJGlq4f8ubUgO50kByRPyt/Cy1Io4PSRsPjxME+YlVaCOx+NIToW7hCsZNFJyTPFFKepRSA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz", + "integrity": "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==", "dev": true, + "license": "MIT", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -8608,14 +8613,11 @@ } }, "node_modules/magic-string": { - "version": "0.30.5", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.5.tgz", - "integrity": "sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==", + "version": "0.30.14", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.14.tgz", + "integrity": "sha512-5c99P1WKTed11ZC0HMJOj6CDIue6F8ySu+bJL+85q1zBEIY8IklrJ1eiKC2NDRh3Ct3FcvmJPyQHb9erXMTJNw==", "dependencies": { - "@jridgewell/sourcemap-codec": "^1.4.15" - }, - "engines": { - "node": ">=12" + "@jridgewell/sourcemap-codec": "^1.5.0" } }, "node_modules/map-obj": { @@ -9781,6 +9783,7 @@ "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.0.tgz", "integrity": "sha512-eu38+hdgojoyq63s+yTpN4XMBdt5l8HhMhc4VKLO9KM5caLIBvUm4thi7fFaxyTmCKeNnXZ5pAlBwCUnhA09uw==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" } @@ -11960,17 +11963,18 @@ "dev": true }, "node_modules/sirv": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/sirv/-/sirv-2.0.4.tgz", - "integrity": "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.0.tgz", + "integrity": "sha512-BPwJGUeDaDCHihkORDchNyyTvWFhcusy1XMmhEVTQTwGeybFbp8YEmB+njbPnth1FibULBSBVwCQni25XlCUDg==", "dev": true, + "license": "MIT", "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", "totalist": "^3.0.0" }, "engines": { - "node": ">= 10" + "node": ">=18" } }, "node_modules/slash": { @@ -12642,9 +12646,9 @@ } }, "node_modules/svelte-hmr": { - "version": "0.15.3", - "resolved": "https://registry.npmjs.org/svelte-hmr/-/svelte-hmr-0.15.3.tgz", - "integrity": "sha512-41snaPswvSf8TJUhlkoJBekRrABDXDMdpNpT2tfHIv4JuhgvHqLMhEPGtaQn0BmbNSTkuz2Ed20DF2eHw0SmBQ==", + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/svelte-hmr/-/svelte-hmr-0.16.0.tgz", + "integrity": "sha512-Gyc7cOS3VJzLlfj7wKS0ZnzDVdv3Pn2IuVeJPk9m2skfhcu5bq3wtIZyQGggr7/Iim5rH5cncyQft/kRLupcnA==", "dev": true, "engines": { "node": "^12.20 || ^14.13.1 || >= 16" @@ -13064,6 +13068,7 @@ "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz", "integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } @@ -13395,15 +13400,14 @@ } }, "node_modules/vite": { - "version": "5.4.0", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.0.tgz", - "integrity": "sha512-5xokfMX0PIiwCMCMb9ZJcMyh5wbBun0zUzKib+L65vAZ8GY9ePZMXxFrHbr/Kyll2+LSCY7xtERPpxkBDKngwg==", + "version": "5.4.11", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.11.tgz", + "integrity": "sha512-c7jFQRklXua0mTzneGW9QVyxFjUgwcihC4bXEtujIo2ouWCe1Ajt/amn2PCxYnhYfd5k09JX3SB7OYWFKYqj8Q==", "dev": true, - "license": "MIT", "dependencies": { "esbuild": "^0.21.3", - "postcss": "^8.4.40", - "rollup": "^4.13.0" + "postcss": "^8.4.43", + "rollup": "^4.20.0" }, "bin": { "vite": "bin/vite.js" diff --git a/frontend/package.json b/frontend/package.json index 24cf283420..3c4aa926ff 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.434.2", + "version": "1.435.2", "scripts": { "dev": "vite dev", "build": "vite build", @@ -21,10 +21,10 @@ "@hey-api/openapi-ts": "^0.43.0", "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", - "@sveltejs/adapter-static": "^3.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/adapter-static": "^3.0.6", + "@sveltejs/kit": "^2.9.0", "@sveltejs/package": "^2.2.2", - "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@sveltejs/vite-plugin-svelte": "^3.1.2", "@tailwindcss/forms": "^0.5.3", "@tailwindcss/typography": "^0.5.8", "@types/d3": "^7.4.0", @@ -64,7 +64,7 @@ "tailwindcss": "^3.4.1", "tslib": "^2.6.1", "typescript": "^5.1.3", - "vite": "^5", + "vite": "^5.4.11", "vite-plugin-circular-dependency": "^0.2.1", "vite-plugin-mkcert": "^1.17.5", "yootils": "^0.3.1" diff --git a/frontend/src/app.html b/frontend/src/app.html index 7f93534ddb..718800105a 100644 --- a/frontend/src/app.html +++ b/frontend/src/app.html @@ -27,61 +27,63 @@
-
-
+
+
+
-
- - - - - + + + + + - - - - - -
- Loading... + /> + + + + + +
+ Loading... +
%sveltekit.body% diff --git a/frontend/src/lib/components/AuthSettings.svelte b/frontend/src/lib/components/AuthSettings.svelte new file mode 100644 index 0000000000..56f6d625de --- /dev/null +++ b/frontend/src/lib/components/AuthSettings.svelte @@ -0,0 +1,255 @@ + + +
+ + SSO + OAuth + SCIM/SAML + +
+ +
+ {#if tab === 'sso'} + {#if !$enterpriseLicense || $enterpriseLicense.endsWith('_pro')} + + Without EE, the number of SSO users is limited to 10. SCIM/SAML is available on EE + + {/if} + +
+
+ When at least one of the below options is set, users will be able to login to Windmill via + their third-party account. +
To test SSO, the recommended workflow is to to save the settings and try to login in + an incognito window. + Learn more +
+
+ + + + + + + + + + + + {#each Object.keys(oauths) as k} + {#if !['authelia', 'authentik', 'google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'keycloak', 'slack', 'kanidm', 'zitadel'].includes(k) && 'login_config' in oauths[k]} + {#if oauths[k]} +
+
+ + + { + delete oauths[k] + oauths = { ...oauths } + }} + /> +
+
+ + + + {#if !windmillBuiltins.includes(k) && k != 'slack'} + + {/if} +
+
+ {/if} + {/if} + {/each} +
+
+ + +
+
+ +
+ {:else if tab === 'oauth'} +
+ When one of the below options is set, you will be able to create a specific resource + containing a token automatically generated by the third-party provider. +
+ To test it after setting an oauth client, go to the Resources menu and create a new one of the + type of your oauth client (i.e. a 'github' resource if you set Github OAuth). +
Learn more +
+
+ +
+ + {#each Object.keys(oauths) as k} + {#if oauths[k] && !('login_config' in oauths[k])} + {#if !['slack'].includes(k) && oauths[k]} +
+
+ + + { + delete oauths[k] + oauths = { ...oauths } + }} + /> +
+
+ + + {#if !windmillBuiltins.includes(k) && k != 'slack'} + + {/if} + {#if k == 'snowflake_oauth'} + + {/if} +
+
+ {/if} + {/if} + {/each} + +
+ + {#if oauth_name == 'custom'} + + {:else} + + {/if} + +
+ {:else if tab == 'scim'} + + {/if} +
diff --git a/frontend/src/lib/components/CronInput.svelte b/frontend/src/lib/components/CronInput.svelte index 3ce3caf385..039cc9409a 100644 --- a/frontend/src/lib/components/CronInput.svelte +++ b/frontend/src/lib/components/CronInput.svelte @@ -250,7 +250,7 @@ {#if !disabled} -
+
diff --git a/frontend/src/lib/components/FlowGraphViewer.svelte b/frontend/src/lib/components/FlowGraphViewer.svelte index 326e1275b6..f69c1c9450 100644 --- a/frontend/src/lib/components/FlowGraphViewer.svelte +++ b/frontend/src/lib/components/FlowGraphViewer.svelte @@ -36,6 +36,8 @@ > {#if notAnonynmous} @@ -947,7 +948,12 @@
{#each flowJobIds?.flowJobs ?? [] as loopJobId, j (loopJobId)} - {#if render} + {#if render && j + subflowsSize + 1 == (flowJobIds?.flowJobs.length ?? 0)} + + {/if} + {#if render && j + subflowsSize + 1 > (flowJobIds?.flowJobs.length ?? 0)}