diff --git a/.github/change-versions-mac.sh b/.github/change-versions-mac.sh index 2257d348ae..50ec3b17ed 100755 --- a/.github/change-versions-mac.sh +++ b/.github/change-versions-mac.sh @@ -16,7 +16,7 @@ sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/ sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml -sed -i '' -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 +sed -i '' -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 # sed -i '' -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml sed -i '' -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile sed -i '' -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile diff --git a/.github/change-versions.sh b/.github/change-versions.sh index 77ecbc6985..699b87b0d7 100755 --- a/.github/change-versions.sh +++ b/.github/change-versions.sh @@ -17,7 +17,7 @@ sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/pac sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml -sed -i -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 +sed -i -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1 # sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile diff --git a/.github/workflows/build-publish-rh-image.yml b/.github/workflows/build-publish-rh-image.yml index 8d4ab5343f..eb904758f0 100644 --- a/.github/workflows/build-publish-rh-image.yml +++ b/.github/workflows/build-publish-rh-image.yml @@ -64,7 +64,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} @@ -81,7 +81,7 @@ jobs: platforms: linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql secrets: | rh_username=${{ secrets.RH_USERNAME }} rh_password=${{ secrets.RH_PASSWORD }} diff --git a/.github/workflows/build-staging-image.yml b/.github/workflows/build-staging-image.yml index 197c219b09..8f54411c1a 100644 --- a/.github/workflows/build-staging-image.yml +++ b/.github/workflows/build-staging-image.yml @@ -62,7 +62,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql tags: | ${{ steps.meta-ee-public.outputs.tags }} labels: | diff --git a/.github/workflows/build_windows_worker_.yml b/.github/workflows/build_windows_worker_.yml index 674a97e85f..d093f99f01 100644 --- a/.github/workflows/build_windows_worker_.yml +++ b/.github/workflows/build_windows_worker_.yml @@ -45,7 +45,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql - name: Rename binary with corresponding architecture run: | diff --git a/.github/workflows/docker-image-rpi4.yml b/.github/workflows/docker-image-rpi4.yml index 71a796f679..f7e8358a17 100644 --- a/.github/workflows/docker-image-rpi4.yml +++ b/.github/workflows/docker-image-rpi4.yml @@ -67,7 +67,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,deno_core + features=embedding,parquet,openidconnect,deno_core,php,mysql tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev ${{ steps.meta-public.outputs.tags }} diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml index 42cc21264b..69bc432e68 100644 --- a/.github/workflows/docker-image.yml +++ b/.github/workflows/docker-image.yml @@ -76,7 +76,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=embedding,parquet,openidconnect,jemalloc,deno_core + features=embedding,parquet,openidconnect,jemalloc,deno_core,dind,php,mysql tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }} ${{ steps.meta-public.outputs.tags }} @@ -138,7 +138,7 @@ jobs: platforms: linux/amd64,linux/arm64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql tags: | ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }} ${{ steps.meta-ee-public.outputs.tags }} @@ -200,7 +200,7 @@ jobs: platforms: linux/amd64 push: true build-args: | - features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka + features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql PYTHON_IMAGE=python:3.12.2-slim-bookworm tags: | ${{ steps.meta-ee-public-py312.outputs.tags }} diff --git a/.github/workflows/publish_windows_worker.yml b/.github/workflows/publish_windows_worker.yml index f6d73a7621..50aff4fc19 100644 --- a/.github/workflows/publish_windows_worker.yml +++ b/.github/workflows/publish_windows_worker.yml @@ -47,7 +47,7 @@ jobs: $env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static" mkdir frontend/build && cd backend New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force - cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka + cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql - name: Rename binary with corresponding architecture run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index e305c73393..25a9e63c99 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,49 @@ # Changelog +## [1.437.1](https://github.com/windmill-labs/windmill/compare/v1.437.0...v1.437.1) (2024-12-10) + + +### Bug Fixes + +* **frontend:** fix newly deployed apps with frontend scripts ([bcd2cfe](https://github.com/windmill-labs/windmill/commit/bcd2cfe674038042f08f22d05286596b3442caf9)) + +## [1.437.0](https://github.com/windmill-labs/windmill/compare/v1.436.0...v1.437.0) (2024-12-10) + + +### Features + +* add ctx.name to apps [#4885](https://github.com/windmill-labs/windmill/issues/4885) ([2387503](https://github.com/windmill-labs/windmill/commit/23875033dd19d6d585f2433fa7b18a703175e329)) + + +### Bug Fixes + +* **parsers:** more robust pwsh param parsing ([#4884](https://github.com/windmill-labs/windmill/issues/4884)) ([7014389](https://github.com/windmill-labs/windmill/commit/7014389d677fba563bec02e08815452b195ecd0a)) +* **python:** do not follow symlinks when copying recursively in overlapping sites-package ([c173e46](https://github.com/windmill-labs/windmill/commit/c173e46a724b50d27416bbc169bf2a57fe68808c)) + +## [1.436.0](https://github.com/windmill-labs/windmill/compare/v1.435.2...v1.436.0) (2024-12-09) + + +### Features + +* add db storage for app inline scripts ([#4837](https://github.com/windmill-labs/windmill/issues/4837)) ([2bc4934](https://github.com/windmill-labs/windmill/commit/2bc4934c4f38ca2631eb99140d89a0b3d3b3a2cf)) +* add otlp support ([#4869](https://github.com/windmill-labs/windmill/issues/4869)) ([6d04744](https://github.com/windmill-labs/windmill/commit/6d047449e200d51f060d4107ff5ad0e10af99a66)) +* **cache:** refurbish fs backed cache ([#4863](https://github.com/windmill-labs/windmill/issues/4863)) ([3c4408e](https://github.com/windmill-labs/windmill/commit/3c4408e3dbf8a37c0b862805b1991af3d39ac054)) +* docker runtime ([2c5d07f](https://github.com/windmill-labs/windmill/commit/2c5d07f3d9069b21b877ea2d085f0a851b1975c1)) +* **frontend:** render new job kinds ([#4864](https://github.com/windmill-labs/windmill/issues/4864)) ([691ef64](https://github.com/windmill-labs/windmill/commit/691ef6468823c99ab129c53ae699de1a91f77c97)) + + +### Bug Fixes + +* fix `flow_node` uniqueness ([#4850](https://github.com/windmill-labs/windmill/issues/4850)) ([667167a](https://github.com/windmill-labs/windmill/commit/667167a022ed31cb67a6c189cc5bd54b09473f4f)) +* handle flow & workspace renames for `flow_node` ([#4861](https://github.com/windmill-labs/windmill/issues/4861)) ([f175158](https://github.com/windmill-labs/windmill/commit/f175158b9ffcc0d9c35938e61dfe3ce56d3b64c2)) + +## [1.435.2](https://github.com/windmill-labs/windmill/compare/v1.435.1...v1.435.2) (2024-12-05) + + +### Bug Fixes + +* job search toast on error ([#4851](https://github.com/windmill-labs/windmill/issues/4851)) ([a99e63f](https://github.com/windmill-labs/windmill/commit/a99e63f5435725c42e38b46933ff3caa345002b5)) + ## [1.435.1](https://github.com/windmill-labs/windmill/compare/v1.435.0...v1.435.1) (2024-12-05) diff --git a/Dockerfile b/Dockerfile index cff245d3e4..ef2f7a721e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -175,9 +175,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools COPY --from=builder /frontend/build /static_frontend COPY --from=builder /windmill/target/release/windmill ${APP}/windmill -COPY --from=denoland/deno:2.0.4 --chmod=755 /usr/bin/deno /usr/bin/deno +COPY --from=denoland/deno:2.1.2 --chmod=755 /usr/bin/deno /usr/bin/deno -COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer diff --git a/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json b/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json index 8fdca62eca..1b1c6b504b 100644 --- a/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json +++ b/backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json @@ -44,7 +44,8 @@ "deploymentcallback", "singlescriptflow", "flowscript", - "flownode" + "flownode", + "appscript" ] } } diff --git a/backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json b/backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json new file mode 100644 index 0000000000..efe58b7a16 --- /dev/null +++ b/backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO app_script (app, hash, lock, code, code_sha256)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8", + "Bpchar", + "Text", + "Text", + "Bpchar" + ] + }, + "nullable": [ + false + ] + }, + "hash": "0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b" +} diff --git a/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json b/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json new file mode 100644 index 0000000000..5c43791411 --- /dev/null +++ b/backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT format('rawscript/%s', code_sha256) as \"path!: String\"\n FROM app_script WHERE id = $1 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "path!: String", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013" +} diff --git a/backend/.sqlx/query-240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0.json b/backend/.sqlx/query-240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0.json new file mode 100644 index 0000000000..134cb58d17 --- /dev/null +++ b/backend/.sqlx/query-240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT \n MAX (created_at) AS last_deploy, \n COUNT (*) AS deploys_count \n , 'python' AS language\n FROM metrics \n WHERE id = 'no_uv_usage_py'\n\n UNION ALL\n \n SELECT \n MAX (created_at) AS last_deploy, \n COUNT (*) AS deploys_count \n , 'ansible' AS language\n FROM metrics \n WHERE id = 'no_uv_usage_ansible'\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "last_deploy", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "deploys_count", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "language", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0" +} diff --git a/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json b/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json index 20d76cbed4..2f80c0c818 100644 --- a/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json +++ b/backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json @@ -34,7 +34,8 @@ "deploymentcallback", "singlescriptflow", "flowscript", - "flownode" + "flownode", + "appscript" ] } } diff --git a/backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json b/backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json new file mode 100644 index 0000000000..f4021d8985 --- /dev/null +++ b/backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json @@ -0,0 +1,27 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Bpchar", + "Text", + "Text", + "Jsonb" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d" +} diff --git a/backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json b/backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json new file mode 100644 index 0000000000..c9cd8219ee --- /dev/null +++ b/backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json @@ -0,0 +1,23 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT policy as \"policy: sqlx::types::Json>\"\n FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "policy: sqlx::types::Json>", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [ + "Text", + "Text" + ] + }, + "nullable": [ + false + ] + }, + "hash": "5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c" +} diff --git a/backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json b/backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json similarity index 74% rename from backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json rename to backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json index 7e1f008dba..1521b9fd2f 100644 --- a/backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json +++ b/backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", + "query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at, worker_group FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at", "describe": { "columns": [ { @@ -32,6 +32,11 @@ "ordinal": 5, "name": "started_at", "type_info": "Timestamptz" + }, + { + "ordinal": 6, + "name": "worker_group", + "type_info": "Varchar" } ], "parameters": { @@ -43,8 +48,9 @@ true, true, false, + false, false ] }, - "hash": "a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7" + "hash": "6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973" } diff --git a/backend/.sqlx/query-3d363466d79075df3f74f946eff43ca89faefca3bcdf2c533425ca3868b0369a.json b/backend/.sqlx/query-6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b.json similarity index 69% rename from backend/.sqlx/query-3d363466d79075df3f74f946eff43ca89faefca3bcdf2c533425ca3868b0369a.json rename to backend/.sqlx/query-6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b.json index e2db39e4c3..27e21028d4 100644 --- a/backend/.sqlx/query-3d363466d79075df3f74f946eff43ca89faefca3bcdf2c533425ca3868b0369a.json +++ b/backend/.sqlx/query-6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT * FROM usr where username = $1 AND workspace_id = $2", + "query": "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n ", "describe": { "columns": [ { @@ -42,6 +42,16 @@ "ordinal": 7, "name": "role", "type_info": "Varchar" + }, + { + "ordinal": 8, + "name": "super_admin", + "type_info": "Bool" + }, + { + "ordinal": 9, + "name": "name", + "type_info": "Varchar" } ], "parameters": { @@ -58,8 +68,10 @@ false, false, false, + true, + false, true ] }, - "hash": "3d363466d79075df3f74f946eff43ca89faefca3bcdf2c533425ca3868b0369a" + "hash": "6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b" } diff --git a/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json b/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json index a3da2f6bc9..f80192980c 100644 --- a/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json +++ b/backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json @@ -28,7 +28,8 @@ "deploymentcallback", "singlescriptflow", "flowscript", - "flownode" + "flownode", + "appscript" ] } } diff --git a/backend/.sqlx/query-78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f.json b/backend/.sqlx/query-78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f.json new file mode 100644 index 0000000000..d1fad59f5a --- /dev/null +++ b/backend/.sqlx/query-78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_py', $1)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f" +} diff --git a/backend/.sqlx/query-8bd028c8b5f8a4d566f89eebc2e63fd04beaf2b0b49e07c7df42ecddd70737f3.json b/backend/.sqlx/query-8bd028c8b5f8a4d566f89eebc2e63fd04beaf2b0b49e07c7df42ecddd70737f3.json new file mode 100644 index 0000000000..f1f7a0b56c --- /dev/null +++ b/backend/.sqlx/query-8bd028c8b5f8a4d566f89eebc2e63fd04beaf2b0b49e07c7df42ecddd70737f3.json @@ -0,0 +1,18 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO resource\n (workspace_id, path, value, resource_type, created_by, edited_at)\n VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = $3, edited_at = now()", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Varchar", + "Varchar", + "Jsonb", + "Varchar", + "Varchar" + ] + }, + "nullable": [] + }, + "hash": "8bd028c8b5f8a4d566f89eebc2e63fd04beaf2b0b49e07c7df42ecddd70737f3" +} diff --git a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json index 7bf8ca9496..29465b533d 100644 --- a/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json +++ b/backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json @@ -28,7 +28,8 @@ "deploymentcallback", "singlescriptflow", "flowscript", - "flownode" + "flownode", + "appscript" ] } } diff --git a/backend/.sqlx/query-d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052.json b/backend/.sqlx/query-d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052.json new file mode 100644 index 0000000000..89ef99521b --- /dev/null +++ b/backend/.sqlx/query-d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_ansible', $1)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052" +} diff --git a/backend/.sqlx/query-dc32e9e81eab9cda88875020acc414271b96d251bcf54c153c45370d917eea9d.json b/backend/.sqlx/query-dc32e9e81eab9cda88875020acc414271b96d251bcf54c153c45370d917eea9d.json deleted file mode 100644 index dc4d68cd63..0000000000 --- a/backend/.sqlx/query-dc32e9e81eab9cda88875020acc414271b96d251bcf54c153c45370d917eea9d.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n WITH existing AS (\n SELECT id FROM flow_node\n WHERE hash = $1 AND path = $2 AND workspace_id = $3\n AND (code IS NOT DISTINCT FROM $4)\n AND (lock IS NOT DISTINCT FROM $5)\n AND (flow IS NOT DISTINCT FROM $6)\n LIMIT 1\n ),\n inserted AS (\n INSERT INTO flow_node (hash, path, workspace_id, code, lock, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT DO NOTHING\n RETURNING id\n )\n SELECT id FROM existing\n UNION ALL\n SELECT id FROM inserted\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8", - "Text", - "Text", - "Text", - "Text", - "Jsonb" - ] - }, - "nullable": [ - null - ] - }, - "hash": "dc32e9e81eab9cda88875020acc414271b96d251bcf54c153c45370d917eea9d" -} diff --git a/backend/.sqlx/query-29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a.json b/backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json similarity index 51% rename from backend/.sqlx/query-29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a.json rename to backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json index a761d91385..450683521d 100644 --- a/backend/.sqlx/query-29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a.json +++ b/backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json @@ -1,10 +1,15 @@ { "db_name": "PostgreSQL", - "query": "SELECT value FROM app_version WHERE id = $1", + "query": "SELECT app_id, value FROM app_version WHERE id = $1", "describe": { "columns": [ { "ordinal": 0, + "name": "app_id", + "type_info": "Int8" + }, + { + "ordinal": 1, "name": "value", "type_info": "Json" } @@ -15,8 +20,9 @@ ] }, "nullable": [ + false, false ] }, - "hash": "29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a" + "hash": "ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b" } diff --git a/backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json b/backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json new file mode 100644 index 0000000000..0d76f9a721 --- /dev/null +++ b/backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "INSERT INTO app_version_lite (id, value) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Jsonb" + ] + }, + "nullable": [] + }, + "hash": "ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602" +} diff --git a/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json b/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json index 4092b60b71..e48e3b041d 100644 --- a/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json +++ b/backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json @@ -48,7 +48,8 @@ "deploymentcallback", "singlescriptflow", "flowscript", - "flownode" + "flownode", + "appscript" ] } } diff --git a/backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json b/backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json new file mode 100644 index 0000000000..49fe5cf6b8 --- /dev/null +++ b/backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "lock", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "code", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index c71bd2960b..d20737ba51 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -647,7 +647,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.2.0", + "fastrand 2.3.0", "hex", "http 0.2.12", "ring 0.17.8", @@ -685,7 +685,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.2.0", + "fastrand 2.3.0", "http 0.2.12", "http-body 0.4.6", "once_cell", @@ -855,7 +855,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "fastrand 2.2.0", + "fastrand 2.3.0", "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", @@ -999,7 +999,7 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba5289ec98f68f28dd809fd601059e6aa908bb8f6108620930828283d4ee23d7" dependencies = [ - "fastrand 2.2.0", + "fastrand 2.3.0", "gloo-timers", "tokio", ] @@ -1085,9 +1085,9 @@ dependencies = [ [[package]] name = "bigdecimal" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f850665a0385e070b64c38d2354e6c104c8479c59868d1e48a0c13ee2c7a1c1" +checksum = "7f31f3af01c5c65a07985c804d3366560e6fa7883d640a122819b14ec327482c" dependencies = [ "autocfg", "libm", @@ -1138,6 +1138,8 @@ dependencies = [ "cexpr", "clang-sys", "itertools 0.13.0", + "log", + "prettyplease 0.2.25", "proc-macro2", "quote", "regex", @@ -1146,6 +1148,24 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "bindgen" +version = "0.71.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" +dependencies = [ + "bitflags 2.6.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "proc-macro2", + "quote", + "regex", + "rustc-hash 2.1.0", + "shlex", + "syn 2.0.90", +] + [[package]] name = "bit-set" version = "0.5.3" @@ -1254,6 +1274,50 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d696c370c750c948ada61c69a0ee2cbbb9c50b1019ddb86d9317157a99c2cae" +[[package]] +name = "bollard" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97ccca1260af6a459d75994ad5acc1651bcabcbdbc41467cc9786519ab854c30" +dependencies = [ + "base64 0.22.1", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http 1.2.0", + "http-body-util", + "hyper 1.5.1", + "hyper-named-pipe", + "hyper-util", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror 2.0.6", + "tokio", + "tokio-util", + "tower-service", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.47.1-rc.27.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f179cfbddb6e77a5472703d4b30436bff32929c0aa8a9008ecf23d1d3cdd0da" +dependencies = [ + "serde", + "serde_repr", + "serde_with", +] + [[package]] name = "borsh" version = "1.5.3" @@ -1491,9 +1555,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc" +checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" dependencies = [ "jobserver", "libc", @@ -1529,9 +1593,9 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.38" +version = "0.4.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1607,9 +1671,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.22" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69371e34337c4c984bbe322360c2547210bf632eb2814bbe78a6e87a2935bd2b" +checksum = "3135e7ec2ef7b10c6ed8950f0f792ed96ee093fa088608f1c76e569722700c84" dependencies = [ "clap_builder", "clap_derive", @@ -1617,9 +1681,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.22" +version = "4.5.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e24c1b4099818523236a8ca881d2b45db98dadfb4625cf6608c12069fcbbde1" +checksum = "30582fc632330df2bd26877bde0c1f4470d57c582bbc070376afcd04d8cb4838" dependencies = [ "anstream", "anstyle", @@ -1641,9 +1705,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.3" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afb84c814227b90d6895e01398aee0d8033c00e7466aca416fb6a8e0eb19d8a7" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" [[package]] name = "cmake" @@ -1737,9 +1801,9 @@ dependencies = [ [[package]] name = "const_format" -version = "0.2.33" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50c655d81ff1114fb0dcdea9225ea9f0cc712a6f8d189378e82bdf62a473a64b" +checksum = "126f97965c8ad46d6d9163268ff28432e8f6a1196a55578867832e3049df63dd" dependencies = [ "const_format_proc_macros", "konst", @@ -1747,9 +1811,9 @@ dependencies = [ [[package]] name = "const_format_proc_macros" -version = "0.2.33" +version = "0.2.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff1a44b93f47b1bac19a27932f5c591e43d1ba357ee4f61526c8a25603f0eb1" +checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744" dependencies = [ "proc-macro2", "quote", @@ -2440,9 +2504,9 @@ dependencies = [ [[package]] name = "deno_ast" -version = "0.42.2" +version = "0.43.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b9d03b1bbeeecdac54367f075d572131736d06c5be3bc49037855bc5ab1bbb" +checksum = "48d00b724e06d2081a141ec1155756a0b465d413d8e2a7515221f61d482eb2ee" dependencies = [ "base64 0.21.7", "deno_media_type", @@ -2481,18 +2545,18 @@ dependencies = [ [[package]] name = "deno_console" -version = "0.171.0" +version = "0.179.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "144108c8bb93b1df2cda4583d9beb8cd4e18798d3e030af8b07d2c55c1e3259b" +checksum = "2e09f2bbb2d842329b602da25dbab5cd4a342f9a8adcb7c02509fc322f796e79" dependencies = [ "deno_core", ] [[package]] name = "deno_core" -version = "0.311.0" +version = "0.321.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e09bd55da542fa1fde753aff617c355b5d782e763ab2a19e4371a56d7844cac" +checksum = "cd2a54cda74cdc187d5fc2d23370a45cf09f912caf566dd1cd24a50157d809c7" dependencies = [ "anyhow", "bincode", @@ -2504,6 +2568,7 @@ dependencies = [ "deno_ops", "deno_unsync", "futures", + "indexmap 2.7.0", "libc", "memoffset", "parking_lot", @@ -2518,19 +2583,20 @@ dependencies = [ "tokio", "url", "v8", + "wasm_dep_analyzer", ] [[package]] name = "deno_core_icudata" -version = "0.0.73" +version = "0.74.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13951ea98c0a4c372f162d669193b4c9d991512de9f2381dd161027f34b26b1" +checksum = "fe4dccb6147bb3f3ba0c7a48e993bfeb999d2c2e47a81badee80e2b370c8d695" [[package]] name = "deno_fetch" -version = "0.195.0" +version = "0.203.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a91340a1d60cebe1392e4b6e1614709ab5fdcfc1f55842561e498d7ef9b2cb9" +checksum = "a18e66bd3bf786e24a8b8bdc97049fa82957b095a5fd1e142545c5a7cdd2272a" dependencies = [ "base64 0.21.7", "bytes", @@ -2540,6 +2606,7 @@ dependencies = [ "deno_tls", "dyn-clone", "error_reporter", + "hickory-resolver", "http 1.2.0", "http-body-util", "hyper 1.5.1", @@ -2550,8 +2617,9 @@ dependencies = [ "rustls-webpki 0.102.8", "serde", "serde_json", + "thiserror 1.0.69", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-socks", "tokio-util", "tower 0.4.13", @@ -2561,9 +2629,9 @@ dependencies = [ [[package]] name = "deno_media_type" -version = "0.1.4" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8978229b82552bf8457a0125aa20863f023619cfc21ebb007b1e571d68fd85b" +checksum = "eaa135b8a9febc9a51c16258e294e268a1276750780d69e46edb31cced2826e4" dependencies = [ "data-url", "serde", @@ -2585,31 +2653,33 @@ dependencies = [ [[package]] name = "deno_net" -version = "0.163.0" +version = "0.171.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b769fd37232a38bf15a3834c9da8cc99409b52d0cf4a936f3117744369f8063" +checksum = "f7b3a51f7b4d5d64d17a7bc6f7495498f20d809930979d21a059d75e850cdea6" dependencies = [ "deno_core", "deno_permissions", "deno_tls", + "hickory-proto", + "hickory-resolver", "pin-project", "rustls-tokio-stream", "serde", "socket2", + "thiserror 1.0.69", "tokio", - "trust-dns-proto", - "trust-dns-resolver", ] [[package]] name = "deno_ops" -version = "0.187.0" +version = "0.197.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e040fd4def8a67538fe38c9955fd970efc9f44284bd69d44f8992a456afd665d" +checksum = "37a8825d92301cf445727c43f17fee2a20fcdf4370004339965156ae7c56c97e" dependencies = [ "proc-macro-rules", "proc-macro2", "quote", + "stringcase", "strum 0.25.0", "strum_macros 0.25.3", "syn 2.0.90", @@ -2618,9 +2688,9 @@ dependencies = [ [[package]] name = "deno_path_util" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4889646c1ce8437a6fde3acb057fd7e2d039e62c61f5063fc125ed1ede114dc6" +checksum = "ff25f6e08e7a0214bbacdd6f7195c7f1ebcd850c87a624e4ff06326b68b42d99" dependencies = [ "percent-encoding", "thiserror 1.0.69", @@ -2629,9 +2699,9 @@ dependencies = [ [[package]] name = "deno_permissions" -version = "0.31.0" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1328c2d1d26cd066ba9d7d5fb3451081219e373342423f78d7a1b73bfac9849" +checksum = "14e822f98185ab3ddf06104b2407681e0008af52361af32f1cd171b7eda5aa59" dependencies = [ "deno_core", "deno_path_util", @@ -2642,6 +2712,7 @@ dependencies = [ "once_cell", "percent-encoding", "serde", + "thiserror 1.0.69", "which 4.4.2", "winapi", ] @@ -2668,9 +2739,9 @@ dependencies = [ [[package]] name = "deno_tls" -version = "0.158.0" +version = "0.166.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22a1abc6bd8af41aa2496ceceef94f4277092c781a9495c437327a17659553a2" +checksum = "688175eed35e7b3053ec114227894ef24786855405d8844058a48bffa997d85a" dependencies = [ "deno_core", "deno_native_certs", @@ -2679,6 +2750,7 @@ dependencies = [ "rustls-tokio-stream", "rustls-webpki 0.102.8", "serde", + "thiserror 1.0.69", "tokio", "webpki-roots 0.26.7", ] @@ -2696,19 +2768,20 @@ dependencies = [ [[package]] name = "deno_url" -version = "0.171.0" +version = "0.179.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68eb6834d66ff13c19633b0e1b621292461b744858130b0a1c51f34a57ba1a03" +checksum = "ad9a108794e505f2b07665e19ff336c1bcba6adcf7182c90c1d3a6c741d7fcd0" dependencies = [ "deno_core", + "thiserror 1.0.69", "urlpattern", ] [[package]] name = "deno_web" -version = "0.202.0" +version = "0.210.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0862246372f5b559b788aa07ca1b9385909d59630251156a54c4d2c5342d1eb" +checksum = "7679087bcc41f7ae3385f8c12d43bc81cfc54cb9b1ef73983d20f5e39fa4e0da" dependencies = [ "async-trait", "base64-simd 0.8.0", @@ -2719,15 +2792,16 @@ dependencies = [ "flate2", "futures", "serde", + "thiserror 1.0.69", "tokio", "uuid 1.11.0", ] [[package]] name = "deno_webidl" -version = "0.171.0" +version = "0.179.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e969b61b740479379eaf303a6900d11f162f9de610640398f89f0aa58abb7c4" +checksum = "5b55d845e3d64f8de7eff67aaa4b6fe1b23bbc2efe967c984f8c64c8dd85fad4" dependencies = [ "deno_core", ] @@ -3194,9 +3268,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "486f806e73c5707928240ddc295403b1b93c96a02038563881c4a2fd84b81ac4" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "ff" @@ -3941,6 +4015,53 @@ dependencies = [ "ureq", ] +[[package]] +name = "hickory-proto" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447afdcdb8afb9d0a852af6dc65d9b285ce720ed7a59e42a8bf2e931c67bc1b5" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand 0.8.5", + "serde", + "thiserror 1.0.69", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.24.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a2e2aba9c389ce5267d31cf1e4dace82390ae276b0b364ea55630b1fa1b44b4" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto", + "ipconfig", + "lru-cache", + "once_cell", + "parking_lot", + "rand 0.8.5", + "resolv-conf", + "serde", + "smallvec", + "thiserror 1.0.69", + "tokio", + "tracing", +] + [[package]] name = "hkdf" version = "0.12.4" @@ -4139,6 +4260,21 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-named-pipe" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73b7d8abf35697b81a825e386fc151e0d503e8cb5fcb93cc8669c376dfd6f278" +dependencies = [ + "hex", + "hyper 1.5.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", + "winapi", +] + [[package]] name = "hyper-rustls" version = "0.24.2" @@ -4169,7 +4305,20 @@ dependencies = [ "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", + "tower-service", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper 1.5.1", + "hyper-util", + "pin-project-lite", + "tokio", "tower-service", ] @@ -4222,6 +4371,21 @@ dependencies = [ "tracing", ] +[[package]] +name = "hyperlocal" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "986c5ce3b994526b3cd75578e62554abd09f0899d6206de48b3e96ab34ccc8c7" +dependencies = [ + "hex", + "http-body-util", + "hyper 1.5.1", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", +] + [[package]] name = "iana-time-zone" version = "0.1.61" @@ -4369,16 +4533,6 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" -[[package]] -name = "idna" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - [[package]] name = "idna" version = "1.0.3" @@ -4692,9 +4846,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.167" +version = "0.2.168" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc" +checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" [[package]] name = "libgit2-sys" @@ -4809,7 +4963,7 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "674883a98273598ac3aad4301724c56734bea90574c5033af067e8f9fb5eb399" dependencies = [ - "prost", + "prost 0.12.6", "prost-types", ] @@ -4916,7 +5070,7 @@ dependencies = [ "rustls-pki-types", "smtp-proto", "tokio", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "webpki-roots 0.26.7", ] @@ -5109,13 +5263,13 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.11" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -5219,7 +5373,7 @@ checksum = "478b0ff3f7d67b79da2b96f56f334431aef65e15ba4b29dd74a4236e29582bdc" dependencies = [ "base64 0.21.7", "bigdecimal", - "bindgen 0.70.1", + "bindgen 0.71.1", "bitflags 2.6.0", "bitvec", "btoi", @@ -5637,6 +5791,90 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab70038c28ed37b97d8ed414b6429d343a8bbf44c9f79ec854f3a643029ba6d7" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 1.0.69", + "tracing", +] + +[[package]] +name = "opentelemetry-appender-tracing" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab5feffc321035ad94088a7e5333abb4d84a8726e54a802e736ce9dd7237e85b" +dependencies = [ + "opentelemetry", + "tracing", + "tracing-core", + "tracing-subscriber", +] + +[[package]] +name = "opentelemetry-otlp" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91cf61a1868dacc576bf2b2a1c3e9ab150af7272909e80085c3173384fe11f76" +dependencies = [ + "async-trait", + "futures-core", + "http 1.2.0", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost 0.13.4", + "thiserror 1.0.69", + "tokio", + "tonic", + "tracing", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6e05acbfada5ec79023c85368af14abd0b307c015e9064d249b2a950ef459a6" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost 0.13.4", + "tonic", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc1b6902ff63b32ef6c489e8048c5e253e2e4a803ea3ea7e783914536eb15c52" + +[[package]] +name = "opentelemetry_sdk" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "231e9d6ceef9b0b2546ddf52335785ce41252bc7474ee8ba05bfad277be13ab8" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "opentelemetry", + "percent-encoding", + "rand 0.8.5", + "serde_json", + "thiserror 1.0.69", + "tokio", + "tokio-stream", + "tracing", +] + [[package]] name = "option-ext" version = "0.2.0" @@ -6264,7 +6502,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.12.6", +] + +[[package]] +name = "prost" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c0fef6c4230e4ccf618a35c59d7ede15dea37de8427500f50aff708806e42ec" +dependencies = [ + "bytes", + "prost-derive 0.13.4", ] [[package]] @@ -6280,13 +6528,26 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "prost-derive" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "157c5a9d7ea5c2ed2d9fb8f495b64759f7816c7eaea54ba3978f0d63000162e3" +dependencies = [ + "anyhow", + "itertools 0.13.0", + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "prost-types" version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" dependencies = [ - "prost", + "prost 0.12.6", ] [[package]] @@ -6393,7 +6654,7 @@ dependencies = [ "rustc-hash 2.1.0", "rustls 0.23.19", "socket2", - "thiserror 2.0.4", + "thiserror 2.0.6", "tokio", "tracing", ] @@ -6412,7 +6673,7 @@ dependencies = [ "rustls 0.23.19", "rustls-pki-types", "slab", - "thiserror 2.0.4", + "thiserror 2.0.6", "tinyvec", "tracing", "web-time", @@ -6420,9 +6681,9 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a626c6807713b15cac82a6acaccd6043c9a5408c24baae07611fec3f243da" +checksum = "52cd4b1eff68bf27940dd39811292c49e007f4d0b4c357358dc9b0197be6b527" dependencies = [ "cfg_aliases", "libc", @@ -6796,7 +7057,7 @@ dependencies = [ "system-configuration 0.6.1", "tokio", "tokio-native-tls", - "tokio-rustls 0.26.0", + "tokio-rustls 0.26.1", "tokio-util", "tower-service", "url", @@ -7046,15 +7307,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.41" +version = "0.38.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f649912bc1495e167a6edee79151c84b1bad49748cb4f1f1167f459f6224f6" +checksum = "f93dc38ecbab2eb790ff964bb77fa94faf256fd3e73285fd7ba0903b76bedb85" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -7564,6 +7825,17 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "serde_repr" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "serde_spanned" version = "0.6.8" @@ -7599,9 +7871,9 @@ dependencies = [ [[package]] name = "serde_v8" -version = "0.220.0" +version = "0.230.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7a65d91d79acc82aa229aeb084f4a39bda269069bc1520df40f679495388e4" +checksum = "b5a783242d2af51d6955cc04bf2b64adb643ab588b61e9573c908a69dabf8c2f" dependencies = [ "num-bigint", "serde", @@ -8248,6 +8520,12 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "stringcase" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04028eeb851ed08af6aba5caa29f2d59a13ed168cee4d6bd753aeefcf1d636b0" + [[package]] name = "stringprep" version = "0.1.5" @@ -8967,7 +9245,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28cce251fcbc87fac86a866eeb0d6c2d536fc16d06f184bb61aeae11aa4cee0c" dependencies = [ "cfg-if", - "fastrand 2.2.0", + "fastrand 2.3.0", "once_cell", "rustix", "windows-sys 0.59.0", @@ -9002,11 +9280,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.4" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f49a1853cf82743e3b7950f77e0f4d622ca36cf4317cba00c767838bac8d490" +checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" dependencies = [ - "thiserror-impl 2.0.4", + "thiserror-impl 2.0.6", ] [[package]] @@ -9022,9 +9300,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.4" +version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8381894bb3efe0c4acac3ded651301ceee58a15d47c2e34885ed1908ad667061" +checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" dependencies = [ "proc-macro2", "quote", @@ -9237,29 +9515,28 @@ dependencies = [ [[package]] name = "tokio" -version = "1.36.0" +version = "1.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "parking_lot", "pin-project-lite", "signal-hook-registry", "socket2", "tokio-macros", "tracing", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", @@ -9314,12 +9591,11 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.26.0" +version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +checksum = "5f6d0975eaace0cf0fcadee4e4aaa5da15b5c079146f2cffb67c113be122bf37" dependencies = [ "rustls 0.23.19", - "rustls-pki-types", "tokio", ] @@ -9337,9 +9613,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4e6ce100d0eb49a2734f8c0812bcd324cf357d21810932c5df6b96ef2b86f1" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" dependencies = [ "futures-core", "pin-project-lite", @@ -9448,6 +9724,39 @@ dependencies = [ "winnow 0.6.20", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes", + "h2 0.4.7", + "http 1.2.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.5.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost 0.13.4", + "rustls-native-certs 0.8.1", + "rustls-pemfile 2.2.0", + "socket2", + "tokio", + "tokio-rustls 0.26.1", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "toolchain_find" version = "0.4.0" @@ -9469,11 +9778,16 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", + "indexmap 1.9.3", "pin-project", "pin-project-lite", + "rand 0.8.5", + "slab", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] @@ -9651,6 +9965,24 @@ dependencies = [ "url", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a971f6058498b5c0f1affa23e7ea202057a7301dbff68e968b2d578bcbd053" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log 0.2.0", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-serde" version = "0.1.3" @@ -9730,54 +10062,6 @@ dependencies = [ "stable_deref_trait", ] -[[package]] -name = "trust-dns-proto" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3119112651c157f4488931a01e586aa459736e9d6046d3bd9105ffb69352d374" -dependencies = [ - "async-trait", - "cfg-if", - "data-encoding", - "enum-as-inner", - "futures-channel", - "futures-io", - "futures-util", - "idna 0.4.0", - "ipnet", - "once_cell", - "rand 0.8.5", - "serde", - "smallvec", - "thiserror 1.0.69", - "tinyvec", - "tokio", - "tracing", - "url", -] - -[[package]] -name = "trust-dns-resolver" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10a3e6c3aff1718b3c73e395d1f35202ba2ffa847c6a62eea0db8fb4cfe30be6" -dependencies = [ - "cfg-if", - "futures-util", - "ipconfig", - "lru-cache", - "once_cell", - "parking_lot", - "rand 0.8.5", - "resolv-conf", - "serde", - "smallvec", - "thiserror 1.0.69", - "tokio", - "tracing", - "trust-dns-proto", -] - [[package]] name = "try-lock" version = "0.2.5" @@ -10096,7 +10380,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" dependencies = [ "form_urlencoded", - "idna 1.0.3", + "idna", "percent-encoding", "serde", ] @@ -10170,11 +10454,11 @@ dependencies = [ [[package]] name = "v8" -version = "0.106.0" +version = "130.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a381badc47c6f15acb5fe0b5b40234162349ed9d4e4fd7c83a7f5547c0fc69c5" +checksum = "2ee0be58935708fa4d7efb970c6cf9f2d9511d24ee24246481a65b6ee167348d" dependencies = [ - "bindgen 0.69.5", + "bindgen 0.70.1", "bitflags 2.6.0", "fslock", "gzip-header", @@ -10356,6 +10640,15 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wasm_dep_analyzer" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f270206a91783fd90625c8bb0d8fbd459d0b1d1bf209b656f713f01ae7c04b8" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "wav" version = "1.0.1" @@ -10474,7 +10767,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "axum", @@ -10515,7 +10808,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "argon2", @@ -10601,7 +10894,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.435.1" +version = "1.437.1" dependencies = [ "base64 0.22.1", "chrono", @@ -10619,7 +10912,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.435.1" +version = "1.437.1" dependencies = [ "chrono", "serde", @@ -10632,7 +10925,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "serde", @@ -10646,7 +10939,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "async-stream", @@ -10672,6 +10965,11 @@ dependencies = [ "magic-crypt", "mail-send", "object_store", + "opentelemetry", + "opentelemetry-appender-tracing", + "opentelemetry-otlp", + "opentelemetry-semantic-conventions", + "opentelemetry_sdk", "pin-project-lite", "prometheus", "quick_cache", @@ -10683,13 +10981,15 @@ dependencies = [ "serde_json", "sha2 0.10.8", "sqlx", - "thiserror 2.0.4", + "thiserror 2.0.6", "tikv-jemalloc-ctl", "tokio", + "tonic", "tracing", "tracing-appender", "tracing-flame", "tracing-loki", + "tracing-opentelemetry", "tracing-subscriber", "uuid 1.11.0", "windmill-macros", @@ -10697,7 +10997,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.435.1" +version = "1.437.1" dependencies = [ "regex", "serde", @@ -10711,7 +11011,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "bytes", @@ -10734,7 +11034,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.435.1" +version = "1.437.1" dependencies = [ "itertools 0.13.0", "lazy_static", @@ -10746,7 +11046,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.435.1" +version = "1.437.1" dependencies = [ "convert_case 0.6.0", "serde", @@ -10755,7 +11055,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "lazy_static", @@ -10767,7 +11067,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "serde_json", @@ -10779,7 +11079,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "gosyn", @@ -10791,7 +11091,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "lazy_static", @@ -10803,7 +11103,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10814,7 +11114,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "itertools 0.13.0", @@ -10825,7 +11125,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "async-recursion", @@ -10843,7 +11143,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -10860,7 +11160,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "lazy_static", @@ -10872,7 +11172,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "lazy_static", @@ -10890,7 +11190,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "getrandom 0.2.15", @@ -10912,7 +11212,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "serde_json", @@ -10922,7 +11222,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "async-recursion", @@ -10938,6 +11238,7 @@ dependencies = [ "hmac", "itertools 0.13.0", "lazy_static", + "opentelemetry", "prometheus", "regex", "reqwest 0.12.9", @@ -10956,7 +11257,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.435.1" +version = "1.437.1" dependencies = [ "wasm-bindgen", "wasm-bindgen-test", @@ -10966,13 +11267,14 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.435.1" +version = "1.437.1" dependencies = [ "anyhow", "async-recursion", "backon", "base64 0.22.1", "bit-vec", + "bollard", "bytes", "chrono", "const_format", @@ -10982,6 +11284,7 @@ dependencies = [ "deno_core", "deno_fetch", "deno_net", + "deno_permissions", "deno_tls", "deno_url", "deno_web", @@ -11002,6 +11305,7 @@ dependencies = [ "object_store", "once_cell", "openidconnect", + "opentelemetry", "pem 3.0.4", "postgres-native-tls", "prometheus", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index af76d0fea9..f65aee1ddf 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.435.1" +version = "1.437.1" authors.workspace = true edition.workspace = true @@ -30,7 +30,7 @@ members = [ ] [workspace.package] -version = "1.435.1" +version = "1.437.1" authors = ["Ruben Fiszel "] edition = "2021" @@ -64,6 +64,10 @@ tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"] sqlx = ["windmill-worker/sqlx"] deno_core = ["windmill-worker/deno_core", "dep:deno_core"] kafka = ["windmill-api/kafka"] +otel = ["windmill-common/otel", "windmill-worker/otel"] +dind = ["windmill-worker/dind"] +php = ["windmill-worker/php"] +mysql = ["windmill-worker/mysql"] [dependencies] anyhow.workspace = true @@ -96,7 +100,6 @@ deno_core = { workspace = true, optional = true } object_store = { workspace = true, optional = true } quote.workspace = true - [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { optional = true, workspace = true } tikv-jemalloc-sys = { optional = true, workspace = true } @@ -139,7 +142,7 @@ windmill-api-client = { path = "./windmill-api-client" } axum = { version = "^0.7", features = ["multipart"] } headers = "^0" hyper = { version = "^1", features = ["full"] } -tokio = { version = "^1", features = ["full", "tracing"] } +tokio = { version = "^1.42.0", features = ["full", "tracing"] } tower = "^0" tower-http = { version = "^0.6", features = ["trace", "cors"] } tower-cookies = "^0.10" @@ -184,15 +187,16 @@ itertools = "^0" regex = "^1" semver = "^1" -deno_fetch = "0.195.0" -deno_tls = "0.158.0" -deno_console = "0.171.0" -deno_url = "0.171.0" -deno_webidl = "0.171.0" -deno_web = "0.202.0" -deno_net = "0.163.0" -deno_core = "0.311.0" -deno_ast = { version = "=0.42.2", features = ["transpiling"] } +deno_fetch = "0.203.0" +deno_tls = "0.166.0" +deno_console = "0.179.0" +deno_url = "0.179.0" +deno_webidl = "0.179.0" +deno_web = "0.210.0" +deno_net = "0.171.0" +deno_core = "0.321.0" +deno_ast = { version = "=0.43.3", features = ["transpiling"] } +deno_permissions = "0.39.0" swc_common = "=0.37.5" swc_ecma_parser = "=0.149.1" @@ -281,6 +285,17 @@ tar = "^0" http = "^1" async-stream = "^0" +opentelemetry = "0.27.0" +tracing-opentelemetry = "0.28.0" +opentelemetry_sdk = { version = "*", features = ["rt-tokio"] } +opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] } +opentelemetry-appender-tracing = "0.27.0" +opentelemetry-semantic-conventions = { version = "*", features = ["semconv_experimental"] } + +bollard = "0.18.1" + +tonic = { version = "^0", features = ["tls-native-roots"] } + tikv-jemallocator = { version = "0.5" } tikv-jemalloc-sys = { version = "^0.5" } tikv-jemalloc-ctl = { version = "^0.5" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index c7e750d0f4..9d6ca729e0 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -8606d98a692d11b09a387c5efbd6b4335c533fd3 \ No newline at end of file +89a221f8b6e0dc431c668ed804066c97b428f6ef diff --git a/backend/migrations/20241202095902_app_script.down.sql b/backend/migrations/20241202095902_app_script.down.sql new file mode 100644 index 0000000000..c493d82176 --- /dev/null +++ b/backend/migrations/20241202095902_app_script.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +DROP TABLE IF EXISTS app_version_lite; +DROP TABLE IF EXISTS app_script; diff --git a/backend/migrations/20241202095902_app_script.up.sql b/backend/migrations/20241202095902_app_script.up.sql new file mode 100644 index 0000000000..5c3e6a2e0b --- /dev/null +++ b/backend/migrations/20241202095902_app_script.up.sql @@ -0,0 +1,26 @@ +-- Add up migration script here +ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'appscript'; + +-- Same as `app_version` but with a "lite" value (w/ `inlineScript.{code,lock}`). +CREATE TABLE app_version_lite ( + id BIGSERIAL PRIMARY KEY, + value JSONB, + FOREIGN KEY (id) REFERENCES app_version (id) ON DELETE CASCADE +); + +GRANT ALL ON app_version_lite TO windmill_user; +GRANT ALL ON app_version_lite TO windmill_admin; + +-- App `inlineScript`. +CREATE TABLE app_script ( + id BIGSERIAL PRIMARY KEY, + app BIGSERIAL NOT NULL, + hash CHAR(64) NOT NULL UNIQUE, -- sha256 of `app`, `lock`, `code`. + lock TEXT, + code TEXT NOT NULL, + code_sha256 CHAR(64) NOT NULL, -- used to retrieve the policy. + FOREIGN KEY (app) REFERENCES app (id) ON DELETE CASCADE +); + +GRANT ALL ON app_script TO windmill_user; +GRANT ALL ON app_script TO windmill_admin; diff --git a/backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql b/backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql new file mode 100644 index 0000000000..7a5d08f3f8 --- /dev/null +++ b/backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql @@ -0,0 +1,2 @@ +-- Add down migration script here +ALTER TABLE flow_node DROP COLUMN hash_v2; diff --git a/backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql b/backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql new file mode 100644 index 0000000000..799f33d341 --- /dev/null +++ b/backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql @@ -0,0 +1,4 @@ +-- Add up migration script here +CREATE SEQUENCE IF NOT EXISTS flow_node_hash_seq; +ALTER TABLE flow_node ALTER COLUMN hash DROP NOT NULL; +ALTER TABLE flow_node ADD COLUMN hash_v2 CHAR(64) NOT NULL UNIQUE DEFAULT to_hex(nextval('flow_node_hash_seq')); diff --git a/backend/migrations/20241206075559_flow_node_unique_2.down.sql b/backend/migrations/20241206075559_flow_node_unique_2.down.sql new file mode 100644 index 0000000000..b6e6db03e6 --- /dev/null +++ b/backend/migrations/20241206075559_flow_node_unique_2.down.sql @@ -0,0 +1,3 @@ +-- Add down migration script here +ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_unique_2; +ALTER TABLE flow_node ADD CONSTRAINT flow_node_hash_v2_key UNIQUE (hash_v2); diff --git a/backend/migrations/20241206075559_flow_node_unique_2.up.sql b/backend/migrations/20241206075559_flow_node_unique_2.up.sql new file mode 100644 index 0000000000..4e8383bd89 --- /dev/null +++ b/backend/migrations/20241206075559_flow_node_unique_2.up.sql @@ -0,0 +1,3 @@ +-- Add up migration script here +ALTER TABLE flow_node ADD CONSTRAINT flow_node_unique_2 UNIQUE (path, workspace_id, hash_v2); +ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_hash_v2_key; diff --git a/backend/migrations/20241210184049_drop_app_lite.down.sql b/backend/migrations/20241210184049_drop_app_lite.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20241210184049_drop_app_lite.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20241210184049_drop_app_lite.up.sql b/backend/migrations/20241210184049_drop_app_lite.up.sql new file mode 100644 index 0000000000..eab39e59de --- /dev/null +++ b/backend/migrations/20241210184049_drop_app_lite.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +delete from app_version_lite; \ No newline at end of file diff --git a/backend/oauth_connect.json b/backend/oauth_connect.json index 0608f0d3c4..b4f45fa486 100644 --- a/backend/oauth_connect.json +++ b/backend/oauth_connect.json @@ -172,5 +172,13 @@ "user-library-read" ] }, + "xero": { + "auth_url": "https://login.xero.com/identity/connect/authorize", + "token_url": "https://identity.xero.com/connect/token", + "scopes": [ + "offline_access", + "accounting.transactions" + ] + }, "snowflake_oauth": {} } diff --git a/backend/parsers/windmill-parser-bash/src/lib.rs b/backend/parsers/windmill-parser-bash/src/lib.rs index 9b62ad3b0c..39f1d2667b 100644 --- a/backend/parsers/windmill-parser-bash/src/lib.rs +++ b/backend/parsers/windmill-parser-bash/src/lib.rs @@ -9,7 +9,7 @@ use regex_lite::Regex; use serde_json::json; -use std::collections::HashMap; +use std::{collections::HashMap, str::FromStr}; use windmill_parser::{Arg, MainArgSignature, Typ}; pub fn parse_bash_sig(code: &str) -> anyhow::Result { @@ -84,36 +84,94 @@ fn parse_bash_file(code: &str) -> anyhow::Result>> { Ok(Some(args)) } +enum ParserState { + Normal, + InSingleQuote, + InDoubleQuote, +} +fn split_pwsh_args(code: &str) -> Vec<&str> { + let mut chars = code.char_indices().peekable(); + let mut state = ParserState::Normal; + let mut splits = vec![]; + let mut last_idx = 0; + while let Some((idx, char)) = chars.next() { + match (&state, char) { + (ParserState::Normal, '\'') => { + state = ParserState::InSingleQuote; + } + (ParserState::Normal, '"') => { + state = ParserState::InDoubleQuote; + } + (ParserState::InSingleQuote, '\'') => { + state = ParserState::Normal; + } + (ParserState::InDoubleQuote, '"') => { + state = ParserState::Normal; + } + (ParserState::Normal, ',') => { + splits.push(&code[last_idx..idx]); + last_idx = idx + 1; // skip the comma + } + _ => {} + } + } + + if last_idx < code.len() { + splits.push(&code[last_idx..]); + } + + splits +} + fn parse_powershell_file(code: &str) -> anyhow::Result>> { let param_wrapper = RE_POWERSHELL_PARAM.captures(code); let mut args = vec![]; if let Some(param_wrapper) = param_wrapper { let param_wrapper = param_wrapper.get(1).unwrap().as_str(); - for cap in RE_POWERSHELL_ARGS.captures_iter(param_wrapper) { - let typ = cap - .get(1) - .map(|x| x.as_str().to_string()) - .unwrap_or("string".to_string()); - let name = cap.get(2).unwrap().as_str().to_string(); - let default = cap - .get(3) - .or(cap.get(4)) - .map(|x| json!(x.as_str().to_string())); + let params = split_pwsh_args(param_wrapper); + for param in params { + if let Some(cap) = RE_POWERSHELL_ARGS.captures(param) { + let typ = cap.get(1).map(|x| x.as_str().to_string()); + let name = cap.get(2).unwrap().as_str().to_string(); - args.push(Arg { - name: name, - typ: match typ.as_str() { - "string" => Typ::Str(None), - "int" | "long" => Typ::Int, - "decimal" | "double" | "single" => Typ::Float, - "datetime" | "DateTime" => Typ::Datetime, - _ => Typ::Str(None), - }, - default: default.clone(), - otyp: None, - has_default: default.is_some(), - oidx: None, - }); + let mut parsed_typ = if let Some(typ) = typ { + match typ.as_str() { + "string" => Some(Typ::Str(None)), + "int" | "long" => Some(Typ::Int), + "decimal" | "double" | "single" => Some(Typ::Float), + "datetime" | "DateTime" => Some(Typ::Datetime), + _ => None, + } + } else { + None + }; + + let default = if let Some(x) = cap.get(3) { + Some(json!(x.as_str().to_string())) + } else if let Some(x) = cap.get(4) { + if parsed_typ.is_none() { + if x.as_str().parse::().is_ok() { + parsed_typ = Some(Typ::Int); + } else if x.as_str().parse::().is_ok() { + parsed_typ = Some(Typ::Float); + } + } + serde_json::Number::from_str(x.as_str()) + .ok() + .map(serde_json::Value::Number) + } else { + None + }; + + args.push(Arg { + name: name, + typ: parsed_typ.unwrap_or(Typ::Str(None)), + default: default.clone(), + otyp: None, + has_default: default.is_some(), + oidx: None, + }); + } } } Ok(Some(args)) @@ -192,4 +250,77 @@ non_required="${5:-}" Ok(()) } + + #[test] + fn test_parse_powershell_sig() -> anyhow::Result<()> { + let code = r#"param($Msg, [string]$Msg2, $Dflt = "default value, with comma", [int]$Nb = 3 , $Nb2 = 5.0, $Nb3 = 5, $Wahoo = $env:WAHOO)"#; + assert_eq!( + parse_powershell_sig(code)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![ + Arg { + otyp: None, + name: "Msg".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None + }, + Arg { + otyp: None, + name: "Msg2".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None + }, + Arg { + otyp: None, + name: "Dflt".to_string(), + typ: Typ::Str(None), + default: Some(json!("default value, with comma")), + has_default: true, + oidx: None + }, + Arg { + otyp: None, + name: "Nb".to_string(), + typ: Typ::Int, + default: Some(json!(3)), + has_default: true, + oidx: None + }, + Arg { + otyp: None, + name: "Nb2".to_string(), + typ: Typ::Float, + default: Some(json!(5.0)), + has_default: true, + oidx: None + }, + Arg { + otyp: None, + name: "Nb3".to_string(), + typ: Typ::Int, + default: Some(json!(5)), + has_default: true, + oidx: None + }, + Arg { + otyp: None, + name: "Wahoo".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false, + oidx: None + } + ], + no_main_func: None, + has_preprocessor: None + } + ); + Ok(()) + } } diff --git a/backend/src/main.rs b/backend/src/main.rs index b9281385f9..f127415da3 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -8,7 +8,7 @@ use anyhow::Context; use monitor::{ - reload_delete_logs_periodically_setting, reload_indexer_config, + load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config, reload_timeout_wait_result_setting, send_current_log_file_to_object_store, send_logs_to_object_store, }; @@ -37,7 +37,7 @@ use windmill_common::{ EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, - OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, @@ -221,11 +221,66 @@ async fn windmill_main() -> anyhow::Result<()> { let hostname = hostname(); - #[cfg(not(feature = "flamegraph"))] - let _guard = windmill_common::tracing_init::initialize_tracing(&hostname); + let mut enable_standalone_indexer: bool = false; + + let mode = std::env::var("MODE") + .map(|x| x.to_lowercase()) + .map(|x| { + if &x == "server" { + println!("Binary is in 'server' mode"); + Mode::Server + } else if &x == "worker" { + tracing::info!("Binary is in 'worker' mode"); + #[cfg(windows)] + { + println!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL"); + } + Mode::Worker + } else if &x == "agent" { + println!("Binary is in 'agent' mode"); + if std::env::var("BASE_INTERNAL_URL").is_err() { + panic!("BASE_INTERNAL_URL is required in agent mode") + } + if std::env::var("JOB_TOKEN").is_err() { + println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table") + } + + #[cfg(not(feature = "enterprise"))] + { + panic!("Agent mode is only available in the EE, ignoring..."); + } + #[cfg(feature = "enterprise")] + Mode::Agent + } else if &x == "indexer" { + tracing::info!("Binary is in 'indexer' mode"); + #[cfg(not(feature = "tantivy"))] + { + eprintln!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features."); + panic!("Indexer mode requires compiling with the tantivy feature flag."); + } + #[cfg(feature = "tantivy")] + Mode::Indexer + } else if &x == "standalone+search"{ + enable_standalone_indexer = true; + println!("Binary is in 'standalone' mode with search enabled"); + Mode::Standalone + } + else { + if &x != "standalone" { + eprintln!("mode not recognized, defaulting to standalone: {x}"); + } else { + println!("Binary is in 'standalone' mode"); + } + Mode::Standalone + } + }) + .unwrap_or_else(|_| { + tracing::info!("Mode not specified, defaulting to standalone"); + Mode::Standalone + }); #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] - tracing::info!("jemalloc enabled"); + println!("jemalloc enabled"); #[cfg(feature = "flamegraph")] let _guard = windmill_common::tracing_init::setup_flamegraph(); @@ -236,13 +291,13 @@ async fn windmill_main() -> anyhow::Result<()> { "cache" => { #[cfg(feature = "embedding")] { - tracing::info!("Caching embedding model..."); + println!("Caching embedding model..."); windmill_api::embeddings::ModelInstance::load_model_files().await?; - tracing::info!("Cached embedding model"); + println!("Cached embedding model"); } #[cfg(not(feature = "embedding"))] { - tracing::warn!("Embeddings are not enabled, ignoring..."); + println!("Embeddings are not enabled, ignoring..."); } cache_hub_scripts(std::env::args().nth(2)).await?; @@ -256,64 +311,6 @@ async fn windmill_main() -> anyhow::Result<()> { _ => {} } - let mut enable_standalone_indexer: bool = false; - - let mode = std::env::var("MODE") - .map(|x| x.to_lowercase()) - .map(|x| { - if &x == "server" { - tracing::info!("Binary is in 'server' mode"); - Mode::Server - } else if &x == "worker" { - tracing::info!("Binary is in 'worker' mode"); - #[cfg(windows)] - { - tracing::warn!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL"); - } - Mode::Worker - } else if &x == "agent" { - tracing::info!("Binary is in 'agent' mode"); - if std::env::var("BASE_INTERNAL_URL").is_err() { - panic!("BASE_INTERNAL_URL is required in agent mode") - } - if std::env::var("JOB_TOKEN").is_err() { - tracing::warn!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table") - } - - #[cfg(not(feature = "enterprise"))] - { - panic!("Agent mode is only available in the EE, ignoring..."); - } - #[cfg(feature = "enterprise")] - Mode::Agent - } else if &x == "indexer" { - tracing::info!("Binary is in 'indexer' mode"); - #[cfg(not(feature = "tantivy"))] - { - tracing::error!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features."); - panic!("Indexer mode requires compiling with the tantivy feature flag."); - } - #[cfg(feature = "tantivy")] - Mode::Indexer - } else if &x == "standalone+search"{ - enable_standalone_indexer = true; - tracing::info!("Binary is in 'standalone' mode with search enabled"); - Mode::Standalone - } - else { - if &x != "standalone" { - tracing::error!("mode not recognized, defaulting to standalone: {x}"); - } else { - tracing::info!("Binary is in 'standalone' mode"); - } - Mode::Standalone - } - }) - .unwrap_or_else(|_| { - tracing::info!("Mode not specified, defaulting to standalone"); - Mode::Standalone - }); - #[allow(unused_mut)] let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer { 0 @@ -325,7 +322,7 @@ async fn windmill_main() -> anyhow::Result<()> { }; if num_workers > 1 { - tracing::warn!( + println!( "We STRONGLY recommend using at most 1 worker per container, use at your own risks" ); } @@ -347,10 +344,26 @@ async fn windmill_main() -> anyhow::Result<()> { IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)) }; - tracing::info!("Connecting to database..."); + println!("Connecting to database..."); let db = windmill_common::connect_db(server_mode, indexer_mode).await?; + + load_otel(&db).await; + tracing::info!("Database connected"); + let environment = load_base_url(&db) + .await + .unwrap_or_else(|_| "local".to_string()) + .trim_start_matches("https://") + .trim_start_matches("http://") + .split(".") + .next() + .unwrap_or_else(|| "local") + .to_string(); + + #[cfg(not(feature = "flamegraph"))] + let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment); + let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await; tracing::info!( @@ -776,6 +789,15 @@ Windmill Community Edition {GIT_VERSION} tracing::error!(error = %e, "Could not reload debug metrics setting"); } }, + OTEL_SETTING => { + tracing::info!("OTEL setting changed, restarting"); + // we wait a bit randomly to avoid having all servers and workers shutdown at same time + let rd_delay = rand::thread_rng().gen_range(0..4); + tokio::time::sleep(Duration::from_secs(rd_delay)).await; + if let Err(e) = tx.send(()) { + tracing::error!(error = %e, "Could not send killpill"); + } + }, REQUEST_SIZE_LIMIT_SETTING => { if server_mode { tracing::info!("Request limit size change detected, killing server expecting to be restarted"); diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index df3b695168..c11d60621e 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -12,7 +12,7 @@ use std::{ use chrono::{NaiveDateTime, Utc}; use futures::{stream::FuturesUnordered, StreamExt}; -use serde::de::DeserializeOwned; +use serde::{de::DeserializeOwned, Deserializer}; use sqlx::{Pool, Postgres}; use tokio::{ join, @@ -40,7 +40,7 @@ use windmill_common::{ EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING, - PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, + OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING, REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING, SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING, }, @@ -58,7 +58,8 @@ use windmill_common::{ }, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, - MONITOR_LOGS_ON_OBJECT_STORE, SERVICE_LOG_RETENTION_SECS, + MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, + SERVICE_LOG_RETENTION_SECS, }; use windmill_queue::cancel_job; use windmill_worker::{ @@ -199,6 +200,65 @@ pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> { Ok(()) } +fn empty_string_as_none<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let option = as serde::Deserialize>::deserialize(deserializer)?; + Ok(option.filter(|s| !s.is_empty())) +} + +#[derive(serde::Deserialize)] +struct OtelSetting { + metrics_enabled: Option, + logs_enabled: Option, + tracing_enabled: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_endpoint: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_headers: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_protocol: Option, + #[serde(default, deserialize_with = "empty_string_as_none")] + otel_exporter_otlp_compression: Option, +} + +pub async fn load_otel(db: &DB) { + let otel = load_value_from_global_settings(db, OTEL_SETTING).await; + if let Ok(v) = otel { + if let Some(v) = v { + let deser = serde_json::from_value::(v); + if let Ok(o) = deser { + let metrics_enabled = o.metrics_enabled.unwrap_or(false); + let logs_enabled = o.logs_enabled.unwrap_or(false); + let tracing_enabled = o.tracing_enabled.unwrap_or(false); + + OTEL_METRICS_ENABLED.store(metrics_enabled, Ordering::Relaxed); + OTEL_LOGS_ENABLED.store(logs_enabled, Ordering::Relaxed); + OTEL_TRACING_ENABLED.store(tracing_enabled, Ordering::Relaxed); + if let Some(endpoint) = o.otel_exporter_otlp_endpoint.as_ref() { + std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint); + } + if let Some(headers) = o.otel_exporter_otlp_headers.as_ref() { + std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", headers); + } + if let Some(protocol) = o.otel_exporter_otlp_protocol { + std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", protocol); + } + if let Some(compression) = o.otel_exporter_otlp_compression { + std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", compression); + } + println!("OTEL settings loaded: tracing ({tracing_enabled}), logs ({logs_enabled}), metrics ({metrics_enabled}), endpoint ({:?}), headers defined: ({})", + o.otel_exporter_otlp_endpoint, o.otel_exporter_otlp_headers.is_some()); + } else { + tracing::error!("Error deserializing otel settings"); + } + } + } else { + tracing::error!("Error loading otel settings: {}", otel.unwrap_err()); + } +} + pub async fn load_tag_per_workspace_enabled(db: &DB) -> error::Result<()> { let metrics_enabled = load_value_from_global_settings(db, DEFAULT_TAGS_PER_WORKSPACE_SETTING).await; @@ -1337,7 +1397,7 @@ pub async fn reload_worker_config( } } -pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { +pub async fn load_base_url(db: &DB) -> error::Result { let q_base_url = load_value_from_global_settings(db, BASE_URL_SETTING).await?; let std_base_url = std::env::var("BASE_URL") @@ -1361,6 +1421,14 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { std_base_url }; + { + let mut l = BASE_URL.write().await; + *l = base_url.clone(); + } + Ok(base_url) +} + +pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?; let oauths = if let Some(q) = q_oauth { @@ -1374,6 +1442,7 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { None }; + let base_url = load_base_url(db).await?; let is_secure = base_url.starts_with("https://"); { @@ -1383,11 +1452,6 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> { .unwrap(); } - { - let mut l = BASE_URL.write().await; - *l = base_url - } - { let mut l = IS_SECURE.write().await; *l = is_secure; diff --git a/backend/tests/worker.rs b/backend/tests/worker.rs index 1498c0168a..aaa1c08f68 100644 --- a/backend/tests/worker.rs +++ b/backend/tests/worker.rs @@ -77,7 +77,11 @@ async fn initialize_tracing() { static ONCE: Once = Once::new(); ONCE.call_once(|| { - let _ = windmill_common::tracing_init::initialize_tracing("test"); + let _ = windmill_common::tracing_init::initialize_tracing( + "test", + &windmill_common::utils::Mode::Standalone, + "test", + ); }); } diff --git a/backend/windmill-api/openapi-deref.yaml b/backend/windmill-api/openapi-deref.yaml index 0e7502d6cd..c8893f824c 100644 --- a/backend/windmill-api/openapi-deref.yaml +++ b/backend/windmill-api/openapi-deref.yaml @@ -8709,6 +8709,9 @@ paths: - identity - deploymentcallback - singlescriptflow + - flowscript + - flownode + - appscript schedule_path: type: string permissioned_as: @@ -9267,6 +9270,9 @@ paths: - identity - deploymentcallback - singlescriptflow + - flowscript + - flownode + - appscript schedule_path: type: string permissioned_as: diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index c30c186f1e..cf6bf8eac4 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.435.1 + version: 1.437.1 title: Windmill API contact: @@ -5467,6 +5467,23 @@ paths: schema: $ref: "#/components/schemas/AppWithLastVersion" + /w/{workspace}/apps/get/lite/{path}: + get: + summary: get app lite by path + operationId: getAppLiteByPath + tags: + - app + parameters: + - $ref: "#/components/parameters/WorkspaceId" + - $ref: "#/components/parameters/ScriptPath" + responses: + "200": + description: app lite details + content: + application/json: + schema: + $ref: "#/components/schemas/AppWithLastVersion" + /w/{workspace}/apps/get/draft/{path}: get: summary: get app by path with draft @@ -5791,6 +5808,8 @@ paths: #flow: flow/ path: type: string + version: + type: integer args: {} raw_code: type: object @@ -5808,6 +5827,8 @@ paths: required: - content - language + id: + type: integer force_viewer_static_fields: type: object force_viewer_one_of_fields: @@ -10964,6 +10985,8 @@ components: "deploymentcallback", "singlescriptflow", "flowscript", + "flownode", + "appscript", ] schedule_path: type: string @@ -11085,6 +11108,8 @@ components: "deploymentcallback", "singlescriptflow", "flowscript", + "flownode", + "appscript", ] schedule_path: type: string @@ -11195,6 +11220,8 @@ components: type: string is_admin: type: boolean + name: + type: string is_super_admin: type: boolean created_at: diff --git a/backend/windmill-api/src/apps.rs b/backend/windmill-api/src/apps.rs index e1a9421252..e7b8f6067a 100644 --- a/backend/windmill-api/src/apps.rs +++ b/backend/windmill-api/src/apps.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; /* * Author: Ruben Fiszel @@ -31,6 +31,7 @@ use axum::{ routing::{delete, get, post}, Router, }; +use futures::future::{FutureExt, TryFutureExt}; use hyper::StatusCode; #[cfg(feature = "parquet")] use itertools::Itertools; @@ -50,13 +51,15 @@ use windmill_audit::ActionKind; #[cfg(feature = "parquet")] use windmill_common::s3_helpers::build_object_store_client; use windmill_common::{ - apps::ListAppQuery, + apps::{AppScriptId, ListAppQuery}, + cache::{self, future::FutureCachedExt}, db::UserDB, error::{to_anyhow, Error, JsonResult, Result}, jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode}, users::username_to_permissioned_as, utils::{ - http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath + http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, + Pagination, StripPath, }, variables::{build_crypt, build_crypt_with_key_suffix}, worker::{to_raw_value, CLOUD_HOSTED}, @@ -71,6 +74,7 @@ pub fn workspaced_service() -> Router { .route("/list", get(list_apps)) .route("/list_search", get(list_search_apps)) .route("/get/p/*path", get(get_app)) + .route("/get/lite/*path", get(get_app_lite)) .route("/get/draft/*path", get(get_app_w_draft)) .route("/secret_of/*path", get(get_secret_id)) .route("/get/v/*id", get(get_app_by_id)) @@ -191,15 +195,16 @@ pub type StaticFields = HashMap>; pub type OneOfFields = HashMap>>; pub type AllowUserResources = Vec; -#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)] +#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)] #[serde(rename_all = "lowercase")] pub enum ExecutionMode { + #[default] Anonymous, Publisher, Viewer, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct PolicyTriggerableInputs { static_inputs: StaticFields, one_of_inputs: OneOfFields, @@ -215,7 +220,7 @@ pub struct S3Input { file_key_regex: String, } -#[derive(Serialize, Deserialize, Debug, Clone)] +#[derive(Serialize, Deserialize, Debug, Clone, Default)] pub struct Policy { pub on_behalf_of: Option, pub on_behalf_of_email: Option, @@ -410,6 +415,33 @@ async fn get_app( Ok(Json(app)) } +async fn get_app_lite( + authed: ApiAuthed, + Extension(user_db): Extension, + Path((w_id, path)): Path<(String, StripPath)>, +) -> JsonResult { + let path = path.to_path(); + let mut tx = user_db.begin(&authed).await?; + + let app_o = sqlx::query_as::<_, AppWithLastVersion>( + "SELECT app.id, app.path, app.summary, app.versions, app.policy, + app.extra_perms, coalesce(app_version_lite.value::json, app_version.value) as value, + app_version.created_at, app_version.created_by, NULL as starred + FROM app, app_version + LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id + WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]", + ) + .bind(path.to_owned()) + .bind(&w_id) + .fetch_optional(&mut *tx) + .await?; + + tx.commit().await?; + + let app = not_found_if_none(app_o, "App", path)?; + Ok(Json(app)) +} + async fn get_app_w_draft( authed: ApiAuthed, Extension(user_db): Extension, @@ -527,12 +559,11 @@ async fn update_app_history( return Ok(()); } - async fn custom_path_exists( Extension(db): Extension, Path((w_id, custom_path)): Path<(String, String)>, ) -> JsonResult { - let exists = + let exists = sqlx::query_scalar!( "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))", custom_path, @@ -583,8 +614,9 @@ async fn get_public_app_by_secret( let app_o = sqlx::query_as::<_, AppWithLastVersion>( "SELECT app.id, app.path, app.summary, app.versions, app.policy, - null as extra_perms, app_version.value, + null as extra_perms, coalesce(app_version_lite.value::json, app_version.value::json) as value, app_version.created_at, app_version.created_by from app, app_version + LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]") .bind(&id) .bind(&w_id) @@ -626,7 +658,6 @@ async fn get_public_app_by_secret( Ok(Json(app)) } - async fn get_public_resource( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, @@ -710,7 +741,6 @@ async fn create_app( } if let Some(custom_path) = &app.custom_path { - require_admin(authed.is_admin, &authed.username)?; let exists = sqlx::query_scalar!( @@ -746,7 +776,9 @@ async fn create_app( app.summary, json!(app.policy), app.draft_only, - app.custom_path, + app.custom_path + .map(|s| if s.is_empty() { None } else { Some(s) }) + .flatten() ) .fetch_one(&mut *tx) .await?; @@ -987,26 +1019,29 @@ async fn update_app( } if let Some(ncustom_path) = &ns.custom_path { - require_admin(authed.is_admin, &authed.username)?; - let exists = sqlx::query_scalar!( - "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))", - ncustom_path, - if *CLOUD_HOSTED { Some(&w_id) } else { None }, - path, - w_id - ) - .fetch_one(&mut *tx) - .await?.unwrap_or(false); + if ncustom_path.is_empty() { + sqlb.set("custom_path", "NULL"); + } else { + let exists = sqlx::query_scalar!( + "SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))", + ncustom_path, + if *CLOUD_HOSTED { Some(&w_id) } else { None }, + path, + w_id + ) + .fetch_one(&mut *tx) + .await?.unwrap_or(false); - if exists { - return Err(Error::BadRequest(format!( - "App with custom path {} already exists", - ncustom_path - ))); + if exists { + return Err(Error::BadRequest(format!( + "App with custom path {} already exists", + ncustom_path + ))); + } + sqlb.set_str("custom_path", ncustom_path); } - sqlb.set_str("custom_path", ncustom_path); } if let Some(mut npolicy) = ns.policy { @@ -1144,6 +1179,10 @@ async fn update_app( #[derive(Debug, Deserialize, Clone)] pub struct ExecuteApp { + /// The app version to execute. Fallback to `path` if not provided. + pub version: Option, + /// The app script id (from the `app_script` table) to execute. + pub id: Option, pub args: HashMap>, // - script: script/ // - flow: flow/ @@ -1206,6 +1245,22 @@ async fn get_on_behalf_details_from_policy_and_authed( Ok((username, permissioned_as, email)) } +/// Convert the triggerables from the old format to the new format. +fn empty_triggerables(mut policy: Policy) -> Policy { + use std::mem::take; + if let Some(triggerables) = take(&mut policy.triggerables) { + let mut triggerables_v2 = take(&mut policy.triggerables_v2).unwrap_or_default(); + for (k, static_inputs) in triggerables.into_iter() { + triggerables_v2.insert( + k, + PolicyTriggerableInputs { static_inputs, ..Default::default() }, + ); + } + policy.triggerables_v2 = Some(triggerables_v2); + } + policy +} + async fn execute_component( OptAuthed(opt_authed): OptAuthed, Extension(db): Extension, @@ -1228,99 +1283,129 @@ async fn execute_component( }; let path = path.to_path(); + let (arc_policy, policy): (Arc, Policy); + let policy_triggerables_default = Default::default(); - let policy = match payload.clone() { + // Two cases here: + // 1. The component is executed from the editor (i.e. in "preview" mode), then: + // - The policy is set to default (in `Viewer` execution mode). + // - The policy triggerables are built by the frontend and retrieved from the request + // payload. + // - In case of inline script, the `RawCode` from the request is pushed as is to the + // job queue. + // 2. Otherwise (i.e. "run" mode): + // - The policy and triggerables are fetched from the database. + // - In case of inline script, if an entry exists in the `app_script` table, push + // an `AppScript` job payload, as in (.1) otherwise. + let (policy, policy_triggerables) = match payload { + // 1. "preview" mode. ExecuteApp { - force_viewer_static_fields: Some(static_fields), - force_viewer_one_of_fields: Some(one_of_fields), + force_viewer_static_fields: Some(static_inputs), + force_viewer_one_of_fields: Some(one_of_inputs), force_viewer_allow_user_resources: Some(allow_user_resources), .. - } => { - let mut hm = HashMap::new(); - - if let Some(path) = payload.path.clone() { - hm.insert( - format!("{}:{path}", payload.component), - PolicyTriggerableInputs { - static_inputs: static_fields, - one_of_inputs: one_of_fields, - allow_user_resources, - }, - ); - } else { - hm.insert( - format!( - "{}:{}", - payload.component, - digest(payload.raw_code.clone().unwrap().content.as_str()) - ), - PolicyTriggerableInputs { - static_inputs: static_fields, - one_of_inputs: one_of_fields, - allow_user_resources, - }, - ); - } - Policy { - execution_mode: ExecutionMode::Viewer, - triggerables: None, - triggerables_v2: Some(hm), - on_behalf_of: None, - on_behalf_of_email: None, - s3_inputs: None, - } - } + } => ( + &Policy { execution_mode: ExecutionMode::Viewer, ..Default::default() }, + &PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources }, + ), + // 2. "run" mode. _ => { - let policy_o = sqlx::query_scalar!( - "SELECT policy from app WHERE path = $1 AND workspace_id = $2", + // Policy is fetched from the database on app `path` and `workspace_id`. + let policy_fut = sqlx::query_scalar!( + "SELECT policy as \"policy: sqlx::types::Json>\" + FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1", path, - &w_id + &w_id, ) .fetch_optional(&db) - .await?; + .map_err(Into::::into) + .map(|policy_o| Result::Ok(not_found_if_none(policy_o?, "App", path)?)) + .map(|policy| Result::Ok(serde_json::from_str(policy?.get())?)) + .map_ok(empty_triggerables); - let policy = not_found_if_none(policy_o, "App", path)?; + // 1. The app `version` is provided: cache the fetched policy. + // 2. Otherwise, always fetch the policy from the database. + let policy = if let Some(id) = payload.version { + let cache = cache::anon!({ u64 => Arc } in "policy" <= 1000); + arc_policy = policy_fut + .map_ok(Arc::new) + .cached(cache, &(id as u64)) + .await?; + &*arc_policy + } else { + policy = policy_fut.await?; + &policy + }; - serde_json::from_value::(policy).map_err(to_anyhow)? + // Compute the path for the triggerables map: + // - flow: `flow/` + // - script: `script/` + // - inline script: `rawscript/` + let path = match &payload { + // flow or script: just use the `payload.path`. + ExecuteApp { path: Some(path), .. } => path, + // inline script: without entry in the `app_script` table. + ExecuteApp { raw_code: Some(raw_code), id: None, .. } => &digest(&raw_code.content), + // inline script: with an entry in the `app_script` table. + ExecuteApp { raw_code: Some(_), id: Some(id), .. } => { + let cache = cache::anon!({ u64 => Arc } in "appscriptpath" <= 10000); + // `id` is unique, cache the result. + &*sqlx::query_scalar!( + "SELECT format('rawscript/%s', code_sha256) as \"path!: String\" + FROM app_script WHERE id = $1 LIMIT 1", + id + ) + .fetch_one(&db) + .map_err(Into::::into) + .map_ok(Arc::new) + .cached(cache, &(*id as u64)) + .await? + } + _ => unreachable!(), + }; + + // Retrieve the triggerables from the policy on `path` or `:`. + let triggerables_v2 = policy + .triggerables_v2 + .as_ref() + .ok_or_else(|| Error::BadRequest(format!("Policy is missing triggerables")))?; + let policy_triggerables = triggerables_v2 + .get(path) // start with `path` in case we can avoid the next` format!`. + .or_else(|| triggerables_v2.get(&format!("{}:{}", payload.component, &path))) + .or(match policy.execution_mode { + ExecutionMode::Viewer => Some(&policy_triggerables_default), + _ => None, + }) + .ok_or_else(|| Error::BadRequest(format!("Path {path} forbidden by policy")))?; + + (policy, policy_triggerables) } }; let (username, permissioned_as, email) = get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?; - let (job_payload, (args, job_id), tag) = match payload { - ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => { - let content = &raw_code.content; - let payload = JobPayload::Code(raw_code.clone()); - let path = digest(content); - let args = build_args( - policy, - &component, - path, - args, - opt_authed.as_ref(), - &user_db, - &db, - &w_id, - ) - .await?; - (payload, args, None) - } - ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => { - let (payload, tag) = get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?; - let args = build_args( - policy, - &component, - path.to_string(), - args, - opt_authed.as_ref(), - &user_db, - &db, - &w_id, - ) - .await?; - (payload, args, tag) - } + let (args, job_id) = build_args( + policy, + policy_triggerables, + payload.args, + opt_authed.as_ref(), + &user_db, + &db, + &w_id, + ) + .await?; + + let (job_payload, tag) = match (payload.path, payload.raw_code, payload.id) { + // flow or script: + (Some(path), None, None) => get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?, + // inline script: in "preview" mode or without entry in the `app_script` table. + (None, Some(raw_code), None) => (JobPayload::Code(raw_code), None), + // inline script: in "run" mode and with an entry in the `app_script` table. + (None, Some(RawCode { language, path, cache_ttl, .. }), Some(id)) => ( + JobPayload::AppScript { id: AppScriptId(id), cache_ttl, language, path }, + None, + ), _ => unreachable!(), }; let tx = windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone()); @@ -1655,9 +1740,12 @@ async fn exists_app( } async fn build_args( - policy: Policy, - component: &str, - path: String, + policy: &Policy, + PolicyTriggerableInputs { + static_inputs, + one_of_inputs, + allow_user_resources, + }: &PolicyTriggerableInputs, mut args: HashMap>, authed: Option<&ApiAuthed>, user_db: &UserDB, @@ -1665,54 +1753,6 @@ async fn build_args( w_id: &str, ) -> Result<(PushArgsOwned, Option)> { let mut job_id: Option = None; - let key = format!("{}:{}", component, &path); - let (static_inputs, one_of_inputs, allow_user_resources) = match policy { - Policy { triggerables_v2: Some(t), .. } => { - let PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources } = t - .get(&key) - .or_else(|| t.get(&path)) - .map(|x| x.clone()) - .or_else(|| { - if matches!(policy.execution_mode, ExecutionMode::Viewer) { - Some(PolicyTriggerableInputs { - static_inputs: HashMap::new(), - one_of_inputs: HashMap::new(), - allow_user_resources: Vec::new(), - }) - } else { - None - } - }) - .ok_or_else(|| { - Error::BadRequest(format!("path {} is not allowed in the app policy", path)) - })?; - - (static_inputs, one_of_inputs, allow_user_resources) - } - Policy { triggerables: Some(t), .. } => { - let static_inputs = t - .get(&key) - .or_else(|| t.get(&path)) - .map(|x| x.clone()) - .or_else(|| { - if matches!(policy.execution_mode, ExecutionMode::Viewer) { - Some(HashMap::new()) - } else { - None - } - }) - .ok_or_else(|| { - Error::BadRequest(format!("path {} is not allowed in the app policy", path)) - })?; - - (static_inputs, HashMap::new(), Vec::new()) - } - _ => Err(Error::BadRequest(format!( - "Policy is missing triggerables for {}", - key - )))?, - }; - let mut safe_args = HashMap::>::new(); // tracing::error!("{:?}", allow_user_resources); @@ -1761,16 +1801,16 @@ async fn build_args( } for (k, v) in one_of_inputs { - if safe_args.contains_key(&k) { + if safe_args.contains_key(k) { continue; } - if let Some(arg_val) = args.get(&k) { + if let Some(arg_val) = args.get(k) { let arg_str = arg_val.get(); let options_str_vec = v.iter().map(|x| x.get()).collect::>(); if options_str_vec.contains(&arg_str) { safe_args.insert(k.to_string(), arg_val.clone()); - args.remove(&k); + args.remove(k); continue; } @@ -1781,7 +1821,7 @@ async fn build_args( .all(|x| options_str_vec.contains(&x.get())) { safe_args.insert(k.to_string(), arg_val.clone()); - args.remove(&k); + args.remove(k); continue; } } diff --git a/backend/windmill-api/src/args.rs b/backend/windmill-api/src/args.rs new file mode 100644 index 0000000000..5952a5ee5f --- /dev/null +++ b/backend/windmill-api/src/args.rs @@ -0,0 +1,469 @@ +use std::collections::HashMap; + +#[cfg(feature = "parquet")] +use crate::job_helpers_ee::get_workspace_s3_resource; +use axum::{ + extract::{FromRequest, FromRequestParts, Query, Request}, + http::{HeaderMap, Uri}, + response::{IntoResponse, Response}, +}; +use bytes::Bytes; +use http::{header::CONTENT_TYPE, request::Parts}; +#[cfg(feature = "parquet")] +use object_store::{Attribute, Attributes}; +use serde::Deserialize; +use serde_json::value::RawValue; +use sqlx::types::JsonRawValue; +#[cfg(feature = "parquet")] +use windmill_common::s3_helpers::build_object_store_client; +use windmill_common::{error::Error, worker::to_raw_value, DB}; +use windmill_queue::PushArgsOwned; + +use crate::db::ApiAuthed; +#[cfg(feature = "parquet")] +use crate::job_helpers_ee::{get_random_file_name, upload_file_internal}; + +pub struct WebhookArgs { + pub args: PushArgsOwned, + pub file_req: Option, +} + +impl WebhookArgs { + #[cfg(not(feature = "parquet"))] + pub async fn to_push_args_owned( + self, + _authed: &ApiAuthed, + _db: &DB, + _w_id: &str, + ) -> Result { + return Err(Error::BadRequest(format!( + "Uploading files requires the parquet feature" + ))); + } + + #[cfg(feature = "parquet")] + pub async fn to_push_args_owned( + mut self, + authed: &ApiAuthed, + db: &DB, + w_id: &str, + ) -> Result { + if let Some(req) = self.file_req { + { + let (_, s3_resource) = + get_workspace_s3_resource(authed, db, None, "", w_id, None).await?; + + if let Some(s3_resource) = s3_resource { + let s3_client = build_object_store_client(&s3_resource).await?; + + let content_type = req + .headers() + .get(CONTENT_TYPE) + .map(|x| x.to_str().ok().map(|x| x.to_string())) + .flatten(); + + let file_extension = content_type + .as_ref() + .map(|mime_str| { + mime_guess::get_mime_extensions_str(mime_str) + .map(|x| x.first().map(|x| x.to_string())) + }) + .flatten() + .flatten(); + + let file_key = get_random_file_name(file_extension); + + let options = Attributes::from_iter(vec![ + ( + Attribute::ContentType, + content_type.unwrap_or("application/octet-stream".to_string()), + ), + (Attribute::ContentDisposition, "inline".to_string()), + ]) + .into(); + + upload_file_internal(s3_client, &file_key, req, options).await?; + + self.args.args.insert( + "body".to_string(), + to_raw_value(&serde_json::json!({ + "s3": &file_key + })), + ); + + return Ok(self.args); + } + } + + return Err(Error::BadRequest(format!( + "You need to connect your workspace to an S3 bucket to upload files" + ))); + } + + Ok(self.args) + } +} + +#[derive(Deserialize)] +pub struct RequestQuery { + pub raw: Option, + pub wrap_body: Option, + pub include_header: Option, +} + +#[axum::async_trait] +impl FromRequest for WebhookArgs +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request( + req: Request, + _state: &S, + ) -> Result { + let (content_type, mut extra, use_raw, wrap_body) = { + let headers_map = req.headers(); + let content_type_header = headers_map.get(CONTENT_TYPE); + let content_type = content_type_header.and_then(|value| value.to_str().ok()); + let uri = req.uri(); + let query = Query::::try_from_uri(uri).unwrap().0; + let mut extra = build_extra(&headers_map, query.include_header); + let query_decode = DecodeQueries::from_uri(uri); + if let Some(DecodeQueries(queries)) = query_decode { + extra.extend(queries); + } + let raw = query.raw.as_ref().is_some_and(|x| *x); + let wrap_body = query.wrap_body.as_ref().is_some_and(|x| *x); + (content_type, extra, raw, wrap_body) + }; + + let no_content_type = content_type.is_none(); + if no_content_type || content_type.unwrap().starts_with("application/json") { + let bytes = Bytes::from_request(req, _state) + .await + .map_err(IntoResponse::into_response)?; + if no_content_type && bytes.is_empty() { + if use_raw { + extra.insert("raw_string".to_string(), to_raw_value(&"".to_string())); + } + let mut args = HashMap::new(); + if wrap_body { + args.insert("body".to_string(), to_raw_value(&serde_json::json!({}))); + } + return Ok(Self { + args: PushArgsOwned { extra: Some(extra), args: args }, + file_req: None, + }); + } + let str = String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; + + PushArgsOwned::from_json(extra, use_raw, wrap_body, str) + .await + .map(|args| Self { args, file_req: None }) + } else if content_type + .unwrap() + .starts_with("application/cloudevents+json") + { + let bytes = Bytes::from_request(req, _state) + .await + .map_err(IntoResponse::into_response)?; + let str = String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; + + PushArgsOwned::from_ce_json(extra, use_raw, str) + .await + .map(|args| Self { args, file_req: None }) + } else if content_type + .unwrap() + .starts_with("application/cloudevents-batch+json") + { + Err( + Error::BadRequest(format!("Cloud events batching is not supported yet")) + .into_response(), + ) + } else if content_type.unwrap().starts_with("text/plain") { + let bytes = Bytes::from_request(req, _state) + .await + .map_err(IntoResponse::into_response)?; + let str = String::from_utf8(bytes.to_vec()) + .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; + extra.insert("raw_string".to_string(), to_raw_value(&str)); + Ok(Self { + args: PushArgsOwned { extra: Some(extra), args: HashMap::new() }, + file_req: None, + }) + } else if content_type + .unwrap() + .starts_with("application/x-www-form-urlencoded") + { + let bytes = Bytes::from_request(req, _state) + .await + .map_err(IntoResponse::into_response)?; + + if use_raw { + let raw_string = String::from_utf8(bytes.to_vec()).map_err(|e| { + Error::BadRequest(format!("invalid utf8: {}", e)).into_response() + })?; + extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); + } + + let payload: HashMap> = serde_urlencoded::from_bytes(&bytes) + .map_err(|e| { + Error::BadRequest(format!("invalid urlencoded data: {}", e)).into_response() + })?; + let payload = payload + .into_iter() + .map(|(k, v)| (k, to_raw_value(&v))) + .collect::>(); + + return Ok(Self { + args: PushArgsOwned { extra: Some(extra), args: payload }, + file_req: None, + }); + } else { + return Ok(Self { + args: PushArgsOwned { extra: None, args: HashMap::new() }, + file_req: Some(req), + }); + + // Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response()) + } + } +} + +lazy_static::lazy_static! { + static ref INCLUDE_HEADERS: Vec = std::env::var("INCLUDE_HEADERS") + .ok().map(|x| x + .split(',') + .map(|s| s.to_string()) + .collect()).unwrap_or_default(); +} + +pub fn build_extra( + headers: &HeaderMap, + include_header: Option, +) -> HashMap> { + let mut args = HashMap::new(); + let whitelist = include_header + .map(|s| s.split(",").map(|s| s.to_string()).collect::>()) + .unwrap_or_default(); + + whitelist + .iter() + .chain(INCLUDE_HEADERS.iter()) + .for_each(|h| { + if let Some(v) = headers.get(h) { + args.insert( + h.to_string().to_lowercase().replace('-', "_"), + to_raw_value(&v.to_str().unwrap().to_string()), + ); + } + }); + args +} + +#[derive(Deserialize)] +pub struct IncludeQuery { + pub include_query: Option, +} + +pub struct DecodeQueries(pub HashMap>); + +#[axum::async_trait] +impl FromRequestParts for DecodeQueries +where + S: Send + Sync, +{ + type Rejection = Response; + + async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { + Ok(DecodeQueries::from_uri(&parts.uri).unwrap_or_else(|| DecodeQueries(HashMap::new()))) + } +} + +impl DecodeQueries { + pub fn from_uri(uri: &Uri) -> Option { + let query = uri.query(); + if query.is_none() { + return None; + } + let query = query.unwrap(); + let include_query = serde_urlencoded::from_str::(query) + .map(|x| x.include_query) + .ok() + .flatten() + .unwrap_or_default(); + let parse_query_args = include_query + .split(",") + .map(|s| s.to_string()) + .collect::>(); + let mut args = HashMap::new(); + if !parse_query_args.is_empty() { + let queries = + serde_urlencoded::from_str::>(query).unwrap_or_default(); + parse_query_args.iter().for_each(|h| { + if let Some(v) = queries.get(h) { + args.insert(h.to_string(), to_raw_value(v)); + } + }); + } + Some(DecodeQueries(args)) + } +} + +// impl<'c> PushArgs<'c> { +// pub fn insert, V: Into>>(&mut self, k: K, v: V) { +// self.extra.insert(k.into(), v.into()); +// } +// } + +fn restructure_cloudevents_metadata( + mut p: HashMap>, +) -> Result>, Error> { + let data = p + .remove("data") + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + let str = data.to_string(); + + let wrap_body = str.len() > 0 && str.chars().next().unwrap() != '{'; + + if wrap_body { + let args = serde_json::from_str::>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + let mut hm = HashMap::new(); + hm.insert("body".to_string(), args); + hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p)); + Ok(hm) + } else { + let mut hm = serde_json::from_str::>>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? + .unwrap_or_else(HashMap::new); + hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p)); + Ok(hm) + } +} + +trait PushArgsOwnedExt: Sized { + async fn from_json( + extra: HashMap>, + use_raw: bool, + force_wrap_body: bool, + str: String, + ) -> Result; + + async fn from_ce_json( + extra: HashMap>, + use_raw: bool, + str: String, + ) -> Result; +} + +impl PushArgsOwnedExt for PushArgsOwned { + async fn from_json( + mut extra: HashMap>, + use_raw: bool, + force_wrap_body: bool, + str: String, + ) -> Result { + if use_raw { + extra.insert("raw_string".to_string(), to_raw_value(&str)); + } + + let wrap_body = force_wrap_body || str.len() > 0 && str.chars().next().unwrap() != '{'; + + if wrap_body { + let args = serde_json::from_str::>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())? + .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); + let mut hm = HashMap::new(); + hm.insert("body".to_string(), args); + Ok(PushArgsOwned { extra: Some(extra), args: hm }) + } else { + let hm = serde_json::from_str::>>>(&str) + .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())? + .unwrap_or_else(HashMap::new); + Ok(PushArgsOwned { extra: Some(extra), args: hm }) + } + } + + async fn from_ce_json( + mut extra: HashMap>, + use_raw: bool, + str: String, + ) -> Result { + if use_raw { + extra.insert("raw_string".to_string(), to_raw_value(&str)); + } + + let hm = serde_json::from_str::>>(&str).map_err(|e| { + Error::BadRequest(format!("invalid cloudevents+json: {}", e)).into_response() + })?; + let hm = restructure_cloudevents_metadata(hm).map_err(|e| e.into_response())?; + Ok(PushArgsOwned { extra: Some(extra), args: hm }) + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + #[tokio::test] + async fn test_cloudevents_json_payload() { + let r1 = r#" + { + "specversion" : "1.0", + "type" : "com.example.someevent", + "source" : "/mycontext", + "subject": null, + "id" : "C234-1234-1234", + "time" : "2018-04-05T17:31:00Z", + "comexampleextension1" : "value", + "comexampleothervalue" : 5, + "datacontenttype" : "application/json", + "data" : { + "appinfoA" : "abc", + "appinfoB" : 123, + "appinfoC" : true + } + } + "#; + let r2 = r#" + { + "specversion" : "1.0", + "type" : "com.example.someevent", + "source" : "/mycontext", + "subject": null, + "id" : "C234-1234-1234", + "time" : "2018-04-05T17:31:00Z", + "comexampleextension1" : "value", + "comexampleothervalue" : 5, + "datacontenttype" : "application/json", + "data" : 1.5 + } + "#; + let extra = HashMap::new(); + + let a1 = PushArgsOwned::from_ce_json(extra.clone(), false, r1.to_string()) + .await + .expect("Failed to parse the cloudevent"); + let a2 = PushArgsOwned::from_ce_json(extra.clone(), false, r2.to_string()) + .await + .expect("Failed to parse the cloudevent"); + + a1.args.get("WEBHOOK__METADATA__").expect( + "CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs", + ); + assert_eq!( + a2.args + .get("body") + .expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs") + .to_string(), + "1.5" + ); + } +} diff --git a/backend/windmill-api/src/capture.rs b/backend/windmill-api/src/capture.rs index 241469d05a..7043a5c58a 100644 --- a/backend/windmill-api/src/capture.rs +++ b/backend/windmill-api/src/capture.rs @@ -18,9 +18,12 @@ use windmill_common::{ error::{JsonResult, Result}, utils::{not_found_if_none, StripPath}, }; -use windmill_queue::{PushArgs, PushArgsOwned}; +use windmill_queue::PushArgs; -use crate::db::{ApiAuthed, DB}; +use crate::{ + args::WebhookArgs, + db::{ApiAuthed, DB}, +}; const KEEP_LAST: i64 = 8; @@ -86,8 +89,10 @@ pub async fn new_payload( pub async fn update_payload( Extension(db): Extension, Path((w_id, path)): Path<(String, StripPath)>, - args: PushArgsOwned, + args: WebhookArgs, ) -> Result { + let args = args.args; + let mut tx = db.begin().await?; sqlx::query!( diff --git a/backend/windmill-api/src/http_triggers.rs b/backend/windmill-api/src/http_triggers.rs index ca03d5803c..529692fe3b 100644 --- a/backend/windmill-api/src/http_triggers.rs +++ b/backend/windmill-api/src/http_triggers.rs @@ -1,6 +1,7 @@ #[cfg(feature = "parquet")] use crate::job_helpers_ee::get_workspace_s3_resource; use crate::{ + args::WebhookArgs, db::{ApiAuthed, DB}, jobs::{ run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_flow_by_path_internal, @@ -32,7 +33,6 @@ use windmill_common::{ utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath}, worker::{to_raw_value, CLOUD_HOSTED}, }; -use windmill_queue::PushArgsOwned; lazy_static::lazy_static! { static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/:\w+").unwrap(); @@ -531,7 +531,7 @@ async fn route_job( Query(query): Query>, method: http::Method, headers: HeaderMap, - mut args: PushArgsOwned, + args: WebhookArgs, ) -> impl IntoResponse { let route_path = route_path.to_path(); let (trigger, called_path, params, authed) = match get_http_route_trigger( @@ -547,6 +547,14 @@ async fn route_job( Err(e) => return e.into_response(), }; + let mut args = match args + .to_push_args_owned(&authed, &db, &trigger.workspace_id) + .await + { + Ok(args) => args, + Err(e) => return e.into_response(), + }; + #[cfg(not(feature = "parquet"))] if trigger.static_asset_config.is_some() { return error::Error::InternalErr( diff --git a/backend/windmill-api/src/jobs.rs b/backend/windmill-api/src/jobs.rs index 034aaa932c..622a08f985 100644 --- a/backend/windmill-api/src/jobs.rs +++ b/backend/windmill-api/src/jobs.rs @@ -36,6 +36,7 @@ use crate::db::ApiAuthed; use crate::users::get_scope_tags; use crate::utils::content_plain; use crate::{ + args::{DecodeQueries, WebhookArgs}, db::DB, users::{check_scopes, require_owner_of_path, OptAuthed}, utils::require_super_admin, @@ -83,7 +84,7 @@ use windmill_common::{METRICS_DEBUG_ENABLED, METRICS_ENABLED}; use windmill_common::{get_latest_deployed_hash_for_path, BASE_URL}; use windmill_queue::{ cancel_job, get_queued_job, get_result_by_id_from_running_flow, job_is_complete, push, - DecodeQueries, PushArgs, PushArgsOwned, PushIsolationLevel, + PushArgs, PushArgsOwned, PushIsolationLevel, }; #[cfg(feature = "prometheus")] @@ -631,7 +632,10 @@ async fn get_flow_job_debug_info( } } for job_id in job_ids { - let job = GetQuery::new().with_auth(&opt_authed).fetch(&db, job_id, &w_id).await; + let job = GetQuery::new() + .with_auth(&opt_authed) + .fetch(&db, job_id, &w_id) + .await; if let Ok(job) = job { jobs.insert(job.id().to_string(), job); } @@ -832,7 +836,10 @@ impl<'a> GetQuery<'a> { self.check_auth(cjob.as_ref().map(|job| job.created_by.as_str()))?; if self.with_flow { - cjob = resolve_maybe_value(db, workspace_id, self.with_code, cjob, |job| job.raw_flow.as_mut()).await?; + cjob = resolve_maybe_value(db, workspace_id, self.with_code, cjob, |job| { + job.raw_flow.as_mut() + }) + .await?; } if let Some(mut cjob) = cjob { cjob.inner = format_completed_job_result(cjob.inner); @@ -842,14 +849,16 @@ impl<'a> GetQuery<'a> { } async fn fetch(self, db: &DB, job_id: Uuid, workspace_id: &str) -> error::Result { - let cjob = self.fetch_completed(db, job_id, workspace_id) + let cjob = self + .fetch_completed(db, job_id, workspace_id) .await? .map(Job::CompletedJob); match cjob { Some(cjob) => Ok(cjob), None => { - let job_maybe = self.fetch_queued(db, job_id, workspace_id) + let job_maybe = self + .fetch_queued(db, job_id, workspace_id) .await? .map(Job::QueuedJob); not_found_if_none(job_maybe, "Job", job_id.to_string()) @@ -2940,8 +2949,10 @@ pub async fn run_flow_by_path( Extension(user_db): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: PushArgsOwned, + args: WebhookArgs, ) -> error::Result<(StatusCode, String)> { + let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + run_flow_by_path_inner(authed, db, user_db, w_id, flow_path, run_query, args, None).await } @@ -3116,8 +3127,9 @@ pub async fn run_script_by_path( Extension(user_db): Extension, Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: PushArgsOwned, + args: WebhookArgs, ) -> error::Result<(StatusCode, String)> { + let args = args.to_push_args_owned(&authed, &db, &w_id).await?; run_script_by_path_inner( authed, db, @@ -3218,7 +3230,10 @@ pub async fn run_workflow_as_code( i += 1; } - let job = GetQuery::new().without_logs().fetch_queued(&db, job_id, &w_id).await?; + let job = GetQuery::new() + .without_logs() + .fetch_queued(&db, job_id, &w_id) + .await?; if *CLOUD_HOSTED { tracing::info!("workflow_as_code_tracing id {i} "); @@ -3779,11 +3794,13 @@ pub async fn run_wait_result_script_by_path( Extension(db): Extension, Path((w_id, script_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: PushArgsOwned, + args: WebhookArgs, ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; + let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + run_wait_result_script_by_path_internal( db, run_query, @@ -3861,11 +3878,13 @@ pub async fn run_wait_result_script_by_hash( Extension(db): Extension, Path((w_id, script_hash)): Path<(String, ScriptHash)>, Query(run_query): Query, - args: PushArgsOwned, + args: WebhookArgs, ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; + let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + check_queue_too_long(&db, run_query.queue_limit).await?; let hash = script_hash.0; @@ -3945,11 +3964,13 @@ pub async fn run_wait_result_flow_by_path( Extension(db): Extension, Path((w_id, flow_path)): Path<(String, StripPath)>, Query(run_query): Query, - args: PushArgsOwned, + args: WebhookArgs, ) -> error::Result { #[cfg(feature = "enterprise")] check_license_key_valid().await?; + let args = args.to_push_args_owned(&authed, &db, &w_id).await?; + run_wait_result_flow_by_path_internal( db, run_query, flow_path, authed, user_db, args, w_id, None, ) @@ -4692,12 +4713,12 @@ async fn run_preview_flow_job( pub async fn run_job_by_hash( authed: ApiAuthed, Extension(db): Extension, - Extension(user_db): Extension, Path((w_id, script_hash)): Path<(String, ScriptHash)>, Query(run_query): Query, - args: PushArgsOwned, + args: WebhookArgs, ) -> error::Result<(StatusCode, String)> { + let args = args.to_push_args_owned(&authed, &db, &w_id).await?; run_job_by_hash_inner( authed, db, @@ -4747,6 +4768,7 @@ pub async fn run_job_by_hash_inner( let tag = run_query.tag.clone().or(tag); check_tag_available_for_workspace(&w_id, &tag, &authed).await?; + let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into()); let (uuid, tx) = push( diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index 88243789b4..088707327b 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -46,6 +46,7 @@ use windmill_common::error::AppError; mod ai; mod apps; +mod args; mod audit; mod capture; mod concurrency_groups; diff --git a/backend/windmill-api/src/tracing_init.rs b/backend/windmill-api/src/tracing_init.rs index 60dab73810..d099eb1598 100644 --- a/backend/windmill-api/src/tracing_init.rs +++ b/backend/windmill-api/src/tracing_init.rs @@ -29,11 +29,13 @@ impl OnResponse for MyOnResponse { _span: &tracing::Span, ) { if *LOG_REQUESTS { - tracing::info!( - latency = latency.as_millis(), - status = response.status().as_u16(), - "response" - ) + let latency = latency.as_millis(); + let status = response.status().as_u16(); + if response.status().is_success() { + tracing::info!(latency = latency, status = status, "response") + } else { + tracing::error!(latency = latency, status = status, "response") + } } } } @@ -67,7 +69,7 @@ impl MakeSpan for MyMakeSpan { uri = %request.uri(), username = field::Empty, workspace_id = field::Empty, - trace_id = tracing_id, + traceId = tracing_id, email = field::Empty, ) } diff --git a/backend/windmill-api/src/users.rs b/backend/windmill-api/src/users.rs index 8ea93f1073..79cd07a939 100644 --- a/backend/windmill-api/src/users.rs +++ b/backend/windmill-api/src/users.rs @@ -847,6 +847,7 @@ pub struct UserInfo { pub folders_read: Vec, pub folders: Vec, pub folders_owners: Vec, + pub name: Option, } #[derive(FromRow, Serialize)] @@ -1218,6 +1219,7 @@ async fn whoami( workspace_id: w_id, email: email.clone(), username: email, + name: None, is_admin, is_super_admin: is_admin, created_at: chrono::Utc::now(), @@ -1306,22 +1308,30 @@ async fn get_usage( Ok(usage.to_string()) } +#[derive(FromRow, Serialize)] +pub struct User2 { + pub workspace_id: String, + pub email: String, + pub username: String, + pub is_admin: bool, + pub created_at: chrono::DateTime, + pub operator: bool, + pub disabled: bool, + pub role: Option, + pub super_admin: bool, + pub name: Option, +} + async fn get_user(w_id: &str, username: &str, db: &DB) -> Result> { let user = sqlx::query_as!( - User, - "SELECT * FROM usr where username = $1 AND workspace_id = $2", + User2, + "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2 + ", username, w_id ) .fetch_optional(db) .await?; - let is_super_admin = sqlx::query_scalar!( - "SELECT super_admin FROM password WHERE email = $1", - user.as_ref().map(|x| &x.email) - ) - .fetch_optional(db) - .await? - .unwrap_or(false); let groups = get_groups_for_user( &w_id, username, @@ -1339,8 +1349,9 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result for AppScriptId { + fn into(self) -> u64 { + self.0 as u64 + } +} #[derive(Deserialize)] pub struct ListAppQuery { diff --git a/backend/windmill-common/src/cache.rs b/backend/windmill-common/src/cache.rs index 9810e83be5..45fdd0bd82 100644 --- a/backend/windmill-common/src/cache.rs +++ b/backend/windmill-common/src/cache.rs @@ -1,25 +1,149 @@ use crate::error; +use std::future::Future; +use std::hash::Hash; use std::path::{Path, PathBuf}; -use quick_cache::sync::Cache; +use quick_cache::Equivalent; +use serde::{Deserialize, Serialize}; use sqlx::PgExecutor; +pub use const_format::concatcp; +pub use lazy_static::lazy_static; +pub use quick_cache::sync::Cache; + /// Cache directory for windmill server/worker(s). pub const CACHE_DIR: &str = "/tmp/windmill/cache/"; +/// A file-system backed concurrent cache. +pub struct FsBackedCache { + cache: Cache, + root: &'static str, +} + +impl FsBackedCache { + /// Create a new file-system backed cache with `items_capacity` capacity. + /// The cache will be stored in the `root` directory. + pub fn new(root: &'static str, items_capacity: usize) -> Self { + Self { cache: Cache::new(items_capacity), root } + } + + /// Gets or inserts an item in the cache with key `key`. + pub async fn get_or_insert_async<'a, Q, F>(&'a self, key: &Q, with: F) -> error::Result + where + Q: Hash + Equivalent + ToOwned + Copy + Into, + F: Future>, + { + self.cache + .get_or_insert_async( + key, + fs::import_or_insert_with(self.root, (*key).into(), with), + ) + .await + } +} + +/// Like [`lazy_static`]`, but for file-system backed caches. +/// +/// # Example +/// ```rust +/// use windmill_common::make_static; +/// +/// make_static! { +/// /// String cache with a maximum capacity of 1000 items stored in the +/// /// "subdirectory" directory. +/// static ref CACHE: { u64 => String } in "subdirectory" <= 1000; +/// /// Another cache. +/// static ref ANOTHER_CACHE: { u64 => Vec } in "another" <= 100; +/// } +/// ``` +#[macro_export] +macro_rules! make_static { + { $( $(#[$attr:meta])* static ref $name:ident: { $Key:ty => $Val:ty } in $root:literal <= $cap:literal; )+ } => { + $crate::cache::lazy_static! { + $( + $(#[$attr])* + static ref $name: $crate::cache::FsBackedCache<$Key, $Val> = + $crate::cache::FsBackedCache::new( + $crate::cache::concatcp!($crate::cache::CACHE_DIR, $root), + $cap + ); + )+ + } + }; +} + +// re-export: +pub use make_static; + +/// Create an anonymous file-system backed cache for one-time use. +/// +/// # Example +/// ```rust +/// use windmill_common::anon; +/// let cache = anon!({ u64 => String } in "subdirectory" <= 1000); +/// ``` +#[macro_export] +macro_rules! anon { + ({ $Key:ty => $Val:ty } in $root:literal <= $cap:literal) => {{ + $crate::cache::make_static! { + static ref __ANON__: { $Key => $Val } in $root <= $cap; + } + + &__ANON__ + }}; +} + +// re-export: +pub use anon; + +pub mod future { + use super::*; + + /// Extension trait for futures that can be cached. + pub trait FutureCachedExt: + Future> + Sized + { + /// Get or insert the future result in the cache. + /// + /// # Example + /// ```rust + /// use windmill_common::cache::{self, future::FutureCachedExt}; + /// + /// async { + /// let result = std::future::ready(Ok(42)) + /// .cached(cache::anon!({ u64 => u64 } in "test" <= 1), &42) + /// .await; + /// + /// assert_eq!(result.unwrap(), 42); + /// }; + /// ``` + fn cached( + self, + cache: &FsBackedCache, + key: &Q, + ) -> impl Future> + where + Q: Hash + Equivalent + ToOwned + Copy + Into, + { + cache.get_or_insert_async(key, self) + } + } + + impl> + Sized> + FutureCachedExt for F + { + } +} + pub mod flow { use super::*; use crate::flows::{FlowNodeId, FlowValue}; - /// Cache directory for windmill server/worker(s) flow nodes. - pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "flow"); - - lazy_static::lazy_static! { + make_static! { /// Flow node cache. - /// FIXME: This should be a static but [`Cache`] does not have a const constructor. /// FIXME: Use `Arc` for cheap cloning. - static ref CACHE: Cache = Cache::new(1000); + static ref CACHE: { FlowNodeId => Val } in "flow" <= 1000; } /// Flow node cache value. @@ -75,38 +199,35 @@ pub mod flow { // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. CACHE - .get_or_insert_async( - &node, - fs::import_or_insert_with(CACHE_DIR, node.0 as u64, async { - sqlx::query!( - "SELECT \ + .get_or_insert_async(&node, async { + sqlx::query!( + "SELECT \ lock AS \"lock: String\", \ code AS \"code: String\", \ flow::text AS \"flow: Box\" \ FROM flow_node WHERE id = $1 LIMIT 1", - node.0, - ) - .fetch_one(e) - .await - .map_err(Into::into) - .and_then(|r| { - Ok(Val { - lock: r - .lock - .and_then(|x| if x.is_empty() { None } else { Some(x) }), - code: r.code, - flow: match r.flow { - None => None, - Some(flow) => serde_json::from_str(&flow).map_err(|err| { - error::Error::InternalErr(format!( - "Unable to parse flow value: {err:?}" - )) - })?, - }, - }) + node.0, + ) + .fetch_one(e) + .await + .map_err(Into::into) + .and_then(|r| { + Ok(Val { + lock: r + .lock + .and_then(|x| if x.is_empty() { None } else { Some(x) }), + code: r.code, + flow: match r.flow { + None => None, + Some(flow) => serde_json::from_str(&flow).map_err(|err| { + error::Error::InternalErr(format!( + "Unable to parse flow value: {err:?}" + )) + })?, + }, }) - }), - ) + }) + }) .await } @@ -164,14 +285,10 @@ pub mod script { use super::*; use crate::scripts::{ScriptHash, ScriptLang}; - /// Cache directory for windmill server/worker(s) scripts. - pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "script"); - - lazy_static::lazy_static! { + make_static! { /// Scripts cache. - /// FIXME: This should be a static but [`Cache`] does not have a const constructor. /// FIXME: Use `Arc` for cheap cloning. - static ref CACHE: Cache = Cache::new(1000); + static ref CACHE: { ScriptHash => Val } in "script" <= 1000; } /// Script cache value. @@ -197,34 +314,31 @@ pub mod script { // so only one thread will be able to fetch the data from the database and write it to // the file system and cache, hence no race on the file system. CACHE - .get_or_insert_async( - &hash, - fs::import_or_insert_with(CACHE_DIR, hash.0 as u64, async { - sqlx::query!( - "SELECT \ + .get_or_insert_async(&hash, async { + sqlx::query!( + "SELECT \ lock AS \"lock: String\", \ content AS \"code!: String\", language AS \"language: Option\", \ envs AS \"envs: Vec\", \ codebase AS \"codebase: String\" \ FROM script WHERE hash = $1 AND workspace_id = $2 LIMIT 1", - hash.0, - workspace_id, - ) - .fetch_one(e) - .await - .map_err(Into::into) - .map(|r| Val { - lock: r - .lock - .and_then(|x| if x.is_empty() { None } else { Some(x) }), - code: r.code, - language: r.language, - envs: r.envs, - codebase: r.codebase, - }) - }), - ) + hash.0, + workspace_id, + ) + .fetch_one(e) + .await + .map_err(Into::into) + .map(|r| Val { + lock: r + .lock + .and_then(|x| if x.is_empty() { None } else { Some(x) }), + code: r.code, + language: r.language, + envs: r.envs, + codebase: r.codebase, + }) + }) .await } @@ -280,11 +394,99 @@ pub mod script { } } +pub mod app { + use super::*; + use crate::apps::AppScriptId; + + make_static! { + /// App scripts cache. + /// FIXME: Use `Arc` for cheap cloning. + static ref CACHE: { AppScriptId => Val } in "app" <= 1000; + } + + /// App app script cache value. + #[derive(Debug, Clone, Default)] + pub struct Val { + pub lock: Option, + pub code: String, + } + + /// Fetch the app script referenced by `id` from the cache. + /// If not present, import from the file-system cache or fetch it from the database and write + /// it to the file system and cache. + /// This should be preferred over fetching the database directly. + pub async fn fetch_script( + e: impl PgExecutor<'_>, + id: AppScriptId, + ) -> error::Result<(Option, String)> { + // If not present, `get_or_insert_async` will lock the key until the future completes, + // so only one thread will be able to fetch the data from the database and write it to + // the file system and cache, hence no race on the file system. + CACHE + .get_or_insert_async(&id, async { + sqlx::query!( + "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1", + id.0, + ) + .fetch_one(e) + .await + .map_err(Into::into) + .map(|r| Val { + lock: r + .lock + .and_then(|x| if x.is_empty() { None } else { Some(x) }), + code: r.code, + }) + }) + .await + .map(|Val { lock, code }| (lock, code)) + } + + // ---------------------------------------------------------------------------------------------- + // impl `fs::Bundle` for `Val`. + + #[derive(Copy, Clone)] + pub enum Item { + Lock, + Code, + } + + impl fs::Item for Item { + fn path(&self, root: &Path) -> PathBuf { + match self { + Item::Lock => root.join("lock.txt"), + Item::Code => root.join("code.txt"), + } + } + } + + impl fs::Bundle for Val { + type Item = Item; + + fn items() -> &'static [Self::Item] { + &[Item::Lock, Item::Code] + } + + fn import(&mut self, item: Self::Item, data: Vec) -> error::Result<()> { + match item { + Item::Lock => self.lock = Some(String::from_utf8(data)?), + Item::Code => self.code = String::from_utf8(data)?, + } + Ok(()) + } + + fn export(&self, item: Self::Item) -> error::Result>> { + match item { + Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())), + Item::Code => Ok(Some(self.code.as_bytes().to_vec())), + } + } + } +} + mod fs { use super::*; - use std::future::Future; - use std::fs::{self, OpenOptions}; use std::io::{Read, Write}; @@ -363,4 +565,29 @@ mod fs { } Ok(data) } + + // Auto-implement `Bundle` for all `serde` serializable types. + + impl Item for () { + fn path(&self, root: &Path) -> PathBuf { + root.join("self.json") + } + } + + impl Deserialize<'de> + Serialize + Default> Bundle for T { + type Item = (); + + fn items() -> &'static [Self::Item] { + &[()] + } + + fn import(&mut self, _: Self::Item, data: Vec) -> error::Result<()> { + *self = serde_json::from_slice(&data)?; + Ok(()) + } + + fn export(&self, _: Self::Item) -> error::Result>> { + Ok(Some(serde_json::to_vec(self)?)) + } + } } diff --git a/backend/windmill-common/src/db.rs b/backend/windmill-common/src/db.rs index 8c80dcaf96..a669c36173 100644 --- a/backend/windmill-common/src/db.rs +++ b/backend/windmill-common/src/db.rs @@ -80,14 +80,14 @@ impl UserDB { let mut folders_read = folders_read.clone(); folders_read.extend(folders_write.clone()); - tracing::debug!( - "Setting role to {} {:?} {:?} {:?} {:?}", - user, - authed.username(), - authed.groups(), - folders_read, - folders_write - ); + // tracing::debug!( + // "Setting role to {} {:?} {:?} {:?} {:?}", + // user, + // authed.username(), + // authed.groups(), + // folders_read, + // folders_write + // ); let mut tx = self.db.begin().await?; diff --git a/backend/windmill-common/src/flow_status.rs b/backend/windmill-common/src/flow_status.rs index a5c509d6ef..50c7cf0e61 100644 --- a/backend/windmill-common/src/flow_status.rs +++ b/backend/windmill-common/src/flow_status.rs @@ -17,7 +17,7 @@ use crate::flows::FlowValue; const MINUTES: Duration = Duration::from_secs(60); const HOURS: Duration = MINUTES.saturating_mul(60); -pub const MAX_RETRY_ATTEMPTS: u16 = 1000; +pub const MAX_RETRY_ATTEMPTS: u32 = u32::MAX; pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6); pub fn is_retry_default(v: &RetryStatus) -> bool { @@ -48,7 +48,7 @@ pub struct FlowStatus { #[derive(Serialize, Deserialize, Debug, Clone, Default)] #[serde(default)] pub struct RetryStatus { - pub fail_count: u16, + pub fail_count: u32, pub failed_jobs: Vec, } diff --git a/backend/windmill-common/src/flows.rs b/backend/windmill-common/src/flows.rs index 97429cd081..29edd4db30 100644 --- a/backend/windmill-common/src/flows.rs +++ b/backend/windmill-common/src/flows.rs @@ -147,7 +147,7 @@ impl Retry { /// Takes the number of previous retries and returns the interval until the next retry if any. /// /// May return [`Duration::ZERO`] to retry immediately. - pub fn interval(&self, previous_attempts: u16, silent: bool) -> Option { + pub fn interval(&self, previous_attempts: u32, silent: bool) -> Option { let Self { constant, exponential } = self; if previous_attempts < constant.attempts { @@ -178,7 +178,7 @@ impl Retry { self.constant.attempts != 0 || self.exponential.attempts != 0 } - pub fn max_attempts(&self) -> u16 { + pub fn max_attempts(&self) -> u32 { self.constant .attempts .saturating_add(self.exponential.attempts) @@ -194,7 +194,7 @@ impl Retry { #[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)] #[serde(default)] pub struct ConstantDelay { - pub attempts: u16, + pub attempts: u32, pub seconds: u16, } @@ -202,7 +202,7 @@ pub struct ConstantDelay { #[derive(Deserialize, Serialize, Debug, Clone, PartialEq)] #[serde(default)] pub struct ExponentialDelay { - pub attempts: u16, + pub attempts: u32, pub multiplier: u16, pub seconds: u16, pub random_factor: Option, // percentage, defaults to 0 for no jitter @@ -407,6 +407,12 @@ pub enum InputTransform { #[serde(transparent)] pub struct FlowNodeId(pub i64); +impl Into for FlowNodeId { + fn into(self) -> u64 { + self.0 as u64 + } +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct Branch { #[serde(skip_serializing_if = "Option::is_none")] @@ -713,10 +719,14 @@ pub async fn resolve_maybe_value( workspace_id: &str, with_code: bool, maybe: Option, - value_mut: impl FnOnce(&mut T) -> Option<&mut Json>> + value_mut: impl FnOnce(&mut T) -> Option<&mut Json>>, ) -> Result, Error> { - let Some(mut container) = maybe else { return Ok(None); }; - let Some(value) = value_mut(&mut container) else { return Ok(Some(container)); }; + let Some(mut container) = maybe else { + return Ok(None); + }; + let Some(value) = value_mut(&mut container) else { + return Ok(Some(container)); + }; resolve_value(e, workspace_id, &mut value.0, with_code).await?; Ok(Some(container)) } @@ -728,8 +738,9 @@ pub async fn resolve_value( value: &mut Box, with_code: bool, ) -> Result<(), Error> { - let mut val = serde_json::from_str::(value.get()) - .map_err(|err| Error::InternalErr(format!("resolve: Failed to parse flow value: {}", err)))?; + let mut val = serde_json::from_str::(value.get()).map_err(|err| { + Error::InternalErr(format!("resolve: Failed to parse flow value: {}", err)) + })?; for module in &mut val.modules { resolve_module(e, workspace_id, &mut module.value, with_code).await?; } @@ -746,16 +757,29 @@ pub async fn resolve_module( ) -> Result<(), Error> { use FlowModuleValue::*; - let mut val = serde_json::from_str::(value.get()) - .map_err(|err| Error::InternalErr(format!("resolve: Failed to parse flow module value: {}", err)))?; + let mut val = serde_json::from_str::(value.get()).map_err(|err| { + Error::InternalErr(format!( + "resolve: Failed to parse flow module value: {}", + err + )) + })?; match &mut val { FlowScript { .. } => { // In order to avoid an unnecessary `.clone()` of `val`, take ownership of it's content // using `std::mem::replace`. let FlowScript { - input_transforms, id, tag, language, - custom_concurrency_key, concurrent_limit, concurrency_time_window_s, is_trigger - } = std::mem::replace(&mut val, Identity) else { unreachable!() }; + input_transforms, + id, + tag, + language, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + is_trigger, + } = std::mem::replace(&mut val, Identity) + else { + unreachable!() + }; // Load script lock file and code content. let (lock, content) = if !with_code { (Some("...".to_string()), "...".to_string()) @@ -763,22 +787,44 @@ pub async fn resolve_module( cache::flow::fetch_script(e, id).await? }; val = RawScript { - input_transforms, content, lock, path: None, tag, language, custom_concurrency_key, - concurrent_limit, concurrency_time_window_s, is_trigger + input_transforms, + content, + lock, + path: None, + tag, + language, + custom_concurrency_key, + concurrent_limit, + concurrency_time_window_s, + is_trigger, }; - }, - ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => { + } + ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => { resolve_modules(e, workspace_id, modules, modules_node.take(), with_code).await?; - }, + } BranchOne { branches, default, default_node } => { resolve_modules(e, workspace_id, default, default_node.take(), with_code).await?; for branch in branches { - resolve_modules(e, workspace_id, &mut branch.modules, branch.modules_node.take(), with_code).await?; + resolve_modules( + e, + workspace_id, + &mut branch.modules, + branch.modules_node.take(), + with_code, + ) + .await?; } - }, + } BranchAll { branches, .. } => { for branch in branches { - resolve_modules(e, workspace_id, &mut branch.modules, branch.modules_node.take(), with_code).await?; + resolve_modules( + e, + workspace_id, + &mut branch.modules, + branch.modules_node.take(), + with_code, + ) + .await?; } } _ => {} @@ -801,7 +847,13 @@ pub async fn resolve_modules( .map(|flow| flow.modules)?; } for module in modules.iter_mut() { - Box::pin(resolve_module(e, workspace_id, &mut module.value, with_code)).await?; + Box::pin(resolve_module( + e, + workspace_id, + &mut module.value, + with_code, + )) + .await?; } Ok(()) } diff --git a/backend/windmill-common/src/global_settings.rs b/backend/windmill-common/src/global_settings.rs index 7b932d8f50..9dbfd03b0e 100644 --- a/backend/windmill-common/src/global_settings.rs +++ b/backend/windmill-common/src/global_settings.rs @@ -35,8 +35,9 @@ pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui"; pub const DEV_INSTANCE_SETTING: &str = "dev_instance"; pub const JWT_SECRET_SETTING: &str = "jwt_secret"; pub const EMAIL_DOMAIN_SETTING: &str = "email_domain"; +pub const OTEL_SETTING: &str = "otel"; -pub const ENV_SETTINGS: [&str; 51] = [ +pub const ENV_SETTINGS: [&str; 54] = [ "DISABLE_NSJAIL", "MODE", "NUM_WORKERS", @@ -88,4 +89,7 @@ pub const ENV_SETTINGS: [&str; 51] = [ "WORKER_GROUP", "SAML_METADATA", "INSTANCE_IS_DEV", + "OTEL_METRICS", + "OTEL_TRACING", + "OTEL_LOGS", ]; diff --git a/backend/windmill-common/src/jobs.rs b/backend/windmill-common/src/jobs.rs index 5b687b54fb..c8f8470b69 100644 --- a/backend/windmill-common/src/jobs.rs +++ b/backend/windmill-common/src/jobs.rs @@ -14,6 +14,7 @@ pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE"; pub const PREPROCESSOR_FAKE_ENTRYPOINT: &str = "__WM_PREPROCESSOR"; use crate::{ + apps::AppScriptId, error::{self, to_anyhow, Error}, flow_status::{FlowStatus, RestartedFrom}, flows::{FlowNodeId, FlowValue, Retry}, @@ -41,6 +42,7 @@ pub enum JobKind { DeploymentCallback, FlowScript, FlowNode, + AppScript, } #[derive(sqlx::FromRow, Debug, Serialize, Clone)] @@ -278,6 +280,12 @@ pub enum JobPayload { id: FlowNodeId, // flow_node(id). path: String, // flow node inner path (e.g. `outer/branchall-42`). }, + AppScript { + id: AppScriptId, // app_script(id). + path: Option, + language: ScriptLang, + cache_ttl: Option, + }, Code(RawCode), Dependencies { path: String, diff --git a/backend/windmill-common/src/lib.rs b/backend/windmill-common/src/lib.rs index ad47b198af..4140f6e58b 100644 --- a/backend/windmill-common/src/lib.rs +++ b/backend/windmill-common/src/lib.rs @@ -36,20 +36,20 @@ pub mod job_s3_helpers_ee; pub mod jobs; pub mod more_serde; pub mod oauth2; +pub mod otel_ee; pub mod queue; pub mod s3_helpers; pub mod schedule; pub mod scripts; pub mod server; pub mod stats_ee; +pub mod tracing_init; pub mod users; pub mod utils; pub mod variables; pub mod worker; pub mod workspaces; -pub mod tracing_init; - pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50; pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5; pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5; @@ -87,6 +87,12 @@ lazy_static::lazy_static! { .unwrap_or_else(|| SocketAddr::from(([0, 0, 0, 0], *METRICS_PORT))); pub static ref METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("METRICS_PORT").is_ok() || std::env::var("METRICS_ADDR").is_ok()); + + pub static ref OTEL_METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_METRICS").is_ok()); + pub static ref OTEL_TRACING_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_TRACING").is_ok()); + pub static ref OTEL_LOGS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_LOGS").is_ok()); + + pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false); pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false); diff --git a/backend/windmill-common/src/otel_ee.rs b/backend/windmill-common/src/otel_ee.rs new file mode 100644 index 0000000000..f3ada162f6 --- /dev/null +++ b/backend/windmill-common/src/otel_ee.rs @@ -0,0 +1,58 @@ +/* + * Author: Ruben Fiszel + * Copyright: Windmill Labs, Inc 2022 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use crate::{jobs::QueuedJob, utils::Mode}; +use uuid::Uuid; + +pub fn set_span_parent(_span: &tracing::Span, _rj: &Uuid) {} + +#[cfg(not(all(feature = "otel", feature = "enterprise")))] +pub(crate) type OtelProvider = Option<()>; + +#[cfg(all(feature = "otel", feature = "enterprise"))] +pub(crate) type OtelProvider = Option; + +#[cfg(not(feature = "otel"))] +pub fn otel_ctx() -> () {} + +#[cfg(feature = "otel")] +#[inline(always)] +pub fn otel_ctx() -> opentelemetry::Context { + opentelemetry::Context::current() +} + +#[cfg(not(feature = "otel"))] +impl FutureExt for T {} + +#[cfg(not(feature = "otel"))] +pub trait FutureExt: Sized { + fn with_context(self, _otel_cx: ()) -> Self { + self + } +} + +use tracing_subscriber::EnvFilter; + +pub(crate) fn init_logs_bridge(_mode: &Mode, _hostname: &str, _env: &str) -> Option { + None +} + +#[cfg(all(feature = "otel", feature = "enterprise"))] +pub(crate) fn init_otlp_tracer( + _mode: &Mode, + _hostname: &str, + _env: &str, +) -> Option { + None +} + +pub(crate) fn init_meter_provider(_mode: &Mode, _hostname: &str, _env: &str) -> OtelProvider { + None +} + +pub fn add_root_flow_job_to_otlp(_queued_job: &QueuedJob, _success: bool) {} diff --git a/backend/windmill-common/src/s3_helpers.rs b/backend/windmill-common/src/s3_helpers.rs index a1c5d03888..0274d0f38d 100644 --- a/backend/windmill-common/src/s3_helpers.rs +++ b/backend/windmill-common/src/s3_helpers.rs @@ -122,7 +122,7 @@ pub struct S3Object { #[cfg(feature = "parquet")] pub async fn get_etag_or_empty( - object_store_resource: &mut ObjectStoreResource, + object_store_resource: &ObjectStoreResource, s3_object: S3Object, ) -> Option { let object_store_client = build_object_store_client(object_store_resource).await; diff --git a/backend/windmill-common/src/scripts.rs b/backend/windmill-common/src/scripts.rs index 8176edaa0c..75e4e1e0c0 100644 --- a/backend/windmill-common/src/scripts.rs +++ b/backend/windmill-common/src/scripts.rs @@ -77,6 +77,12 @@ impl ScriptLang { #[sqlx(transparent)] pub struct ScriptHash(pub i64); +impl Into for ScriptHash { + fn into(self) -> u64 { + self.0 as u64 + } +} + #[derive(PartialEq, sqlx::Type)] #[sqlx(transparent, no_pg_array)] pub struct ScriptHashes(pub Vec); diff --git a/backend/windmill-common/src/tracing_init.rs b/backend/windmill-common/src/tracing_init.rs index 13b345e36e..897281007d 100644 --- a/backend/windmill-common/src/tracing_init.rs +++ b/backend/windmill-common/src/tracing_init.rs @@ -7,13 +7,23 @@ */ use const_format::concatcp; + +use std::{ + collections::HashMap, + sync::{Arc, RwLock}, +}; +use tracing::{level_filters::LevelFilter, Event}; use tracing_appender::non_blocking::{NonBlockingBuilder, WorkerGuard}; +use tracing_subscriber::layer::Context; use tracing_subscriber::{ + filter::Targets, fmt::{format, Layer}, prelude::*, EnvFilter, }; +use crate::utils::Mode; + fn json_layer() -> Layer> { tracing_subscriber::fmt::layer() .json() @@ -34,17 +44,40 @@ pub const LOGS_SERVICE: &str = "logs/services/"; pub const TMP_WINDMILL_LOGS_SERVICE: &str = concatcp!("/tmp/windmill/", LOGS_SERVICE); -pub fn initialize_tracing(hostname: &str) -> WorkerGuard { +pub fn initialize_tracing( + hostname: &str, + mode: &Mode, + environment: &str, +) -> (WorkerGuard, crate::otel_ee::OtelProvider) { let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into()); - if std::env::var("RUST_LOG").is_ok_and(|x| x == "debug" || x == "info") { + let rust_log_env = std::env::var("RUST_LOG"); + if rust_log_env + .as_ref() + .is_ok_and(|x| x == "debug" || x == "info") + { std::env::set_var( "RUST_LOG", - &format!("windmill={}", std::env::var("RUST_LOG").unwrap()), + &format!("windmill={}", rust_log_env.as_ref().unwrap()), ) } + let default_env_filter = if rust_log_env.is_ok_and(|x| x == "debug") { + LevelFilter::DEBUG + } else { + LevelFilter::INFO + }; + + let meter_provider = crate::otel_ee::init_meter_provider(mode, hostname, environment); + + #[cfg(all(feature = "otel", feature = "enterprise"))] + let opentelemetry = crate::otel_ee::init_otlp_tracer(mode, hostname, environment) + .map(|x| tracing_opentelemetry::layer().with_tracer(x)); + + #[cfg(not(all(feature = "otel", feature = "enterprise")))] + let opentelemetry: Option = None; + + let logs_bridge = crate::otel_ee::init_logs_bridge(&mode, hostname, environment); - let env_filter = EnvFilter::from_default_env(); use tracing_appender::rolling::{RollingFileAppender, Rotation}; let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname); @@ -61,39 +94,59 @@ pub fn initialize_tracing(hostname: &str) -> WorkerGuard { .finish(file_appender); let stdout_and_log_file_writer = std::io::stdout.and(log_file_writer); - let ts_base = tracing_subscriber::registry().with(env_filter); + // let job_logs_filter = tracing_subscriber::filter::Targets::new() + // .with_target("windmill:job_log", tracing::Level::TRACE); - #[cfg(feature = "loki")] - let ts_base = { - let (layer, task) = tracing_loki::builder() - .build_url(reqwest::Url::parse("http://127.0.0.1:3100").unwrap()) - .expect("build loki url"); - tokio::spawn(task); - ts_base.with(layer) - }; + let env_filter = EnvFilter::builder() + .with_default_directive(tracing::level_filters::LevelFilter::ERROR.into()) + .from_env_lossy(); + + let ts_base = tracing_subscriber::registry().with(env_filter); match *JSON_FMT { true => ts_base + .with(logs_bridge) + .with(opentelemetry) + // .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap())) .with( json_layer() .with_writer(stdout_and_log_file_writer) - .flatten_event(true), + .flatten_event(true) + .with_filter( + Targets::new() + .with_target( + "windmill:job_log", + tracing::level_filters::LevelFilter::OFF, + ) + .with_default(default_env_filter), + ), ) .with(CountingLayer::new()) .init(), false => ts_base + .with(logs_bridge) + .with(opentelemetry) + // .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap())) .with( compact_layer() .with_writer(stdout_and_log_file_writer) .with_ansi(style.to_lowercase() != "never") .with_file(true) .with_line_number(true) - .with_target(false), + .with_target(false) + .with_filter( + Targets::new() + .with_target( + "windmill:job_log", + tracing::level_filters::LevelFilter::OFF, + ) + .with_default(default_env_filter), + ), ) .with(CountingLayer::new()) .init(), } - _guard + (_guard, meter_provider) } #[cfg(feature = "flamegraph")] @@ -112,13 +165,6 @@ pub fn setup_flamegraph() -> impl Drop { _guard } -use std::{ - collections::HashMap, - sync::{Arc, RwLock}, -}; -use tracing::Event; -use tracing_subscriber::layer::Context; - lazy_static::lazy_static! { pub static ref LOG_COUNTING_BY_MIN: Arc>> = Arc::new(RwLock::new(HashMap::new())); } @@ -144,22 +190,6 @@ impl CountingLayer { } } -// impl CountingLayer { -// pub fn new() -> Self { -// CountingLayer { counter: Arc::new(Mutex::new(LogCounter::new())) } -// } - -// pub fn get_counts(&self) -> (usize, usize) { -// let counter = self.counter.lock().unwrap(); -// (counter.non_error_count, counter.error_count) -// } - -// pub fn reset_counts(&self) { -// let mut counter = self.counter.lock().unwrap(); -// counter.reset(); -// } -// } - pub const LOG_TIMESTAMP_FMT: &str = "%Y-%m-%d-%H-%M"; impl tracing_subscriber::Layer for CountingLayer diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 42c02e9a7f..ba8ed8aa52 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -17,7 +17,9 @@ use std::{ use tokio::sync::RwLock; use windmill_macros::annotations; -use crate::{error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB}; +use crate::{ + error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB, +}; lazy_static::lazy_static! { pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string()); @@ -353,6 +355,11 @@ pub struct SqlAnnotations { pub return_last_result: bool, } +#[annotations("#")] +pub struct BashAnnotations { + pub docker: bool, +} + pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) { if tokio::fs::metadata(&bin_path).await.is_ok() { (true, format!("loaded from local cache: {}\n", bin_path)) @@ -578,8 +585,10 @@ pub fn get_windmill_memory_usage() -> Option { } } -pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(executor: E) -> bool { - use crate::utils::{GIT_VERSION, GIT_SEM_VERSION}; +pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>( + executor: E, +) -> bool { + use crate::utils::{GIT_SEM_VERSION, GIT_VERSION}; // fetch all pings with a different version than self from the last 5 minutes. let pings = sqlx::query_scalar!( diff --git a/backend/windmill-queue/Cargo.toml b/backend/windmill-queue/Cargo.toml index 7ac744a885..8f94de918a 100644 --- a/backend/windmill-queue/Cargo.toml +++ b/backend/windmill-queue/Cargo.toml @@ -43,4 +43,5 @@ bigdecimal.workspace = true axum.workspace = true serde_urlencoded.workspace = true regex.workspace = true -backon.workspace = true \ No newline at end of file +backon.workspace = true +opentelemetry.workspace = true diff --git a/backend/windmill-queue/src/jobs.rs b/backend/windmill-queue/src/jobs.rs index 1e516f4821..e88a7c198c 100644 --- a/backend/windmill-queue/src/jobs.rs +++ b/backend/windmill-queue/src/jobs.rs @@ -10,29 +10,19 @@ use std::{borrow::Borrow, collections::HashMap, sync::Arc, vec}; use anyhow::Context; use async_recursion::async_recursion; -use axum::{ - body::Bytes, - extract::{FromRequest, FromRequestParts, Query}, - http::{request::Parts, Request, Uri}, - response::{IntoResponse, Response}, -}; use chrono::{DateTime, Duration, Utc}; use futures::future::TryFutureExt; use itertools::Itertools; #[cfg(feature = "prometheus")] use prometheus::IntCounter; use regex::Regex; -use reqwest::{ - header::{HeaderMap, CONTENT_TYPE}, - Client, StatusCode, -}; -use serde::{ser::SerializeMap, Deserialize, Serialize}; +use reqwest::Client; +use serde::{ser::SerializeMap, Serialize}; use serde_json::{json, value::RawValue}; use sqlx::{types::Json, FromRow, Pool, Postgres, Transaction}; #[cfg(feature = "benchmark")] use std::time::Instant; use tokio::{sync::RwLock, time::sleep}; -use tracing::{instrument, Instrument}; use ulid::Ulid; use uuid::Uuid; use windmill_audit::audit_ee::{audit_log, AuditAuthor}; @@ -454,7 +444,6 @@ where } } -#[instrument(level = "trace", skip_all)] pub async fn add_completed_job_error( db: &Pool, queued_job: &QueuedJob, @@ -510,7 +499,6 @@ lazy_static::lazy_static! { pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok(); } -#[instrument(level = "trace", skip_all, name = "add_completed_job")] pub async fn add_completed_job( db: &Pool, queued_job: &QueuedJob, @@ -643,9 +631,7 @@ pub async fn add_completed_job( .fetch_one(&mut *tx) .await .map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e:#}")))?; - // tracing::error!("2 {:?}", start.elapsed()); - // add_time!(bench, "add_completed_job query END"); if !queued_job.is_flow_step { if _duration > 500 @@ -1259,7 +1245,6 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync> Ok(()) } -#[instrument(level = "trace", skip_all)] pub async fn handle_maybe_scheduled_job<'c>( db: &Pool, job: &QueuedJob, @@ -2429,7 +2414,6 @@ async fn extract_result_from_job_result( } } -#[instrument(level = "trace", skip_all)] pub async fn delete_job<'c>( mut tx: Transaction<'c, Postgres>, w_id: &str, @@ -2576,340 +2560,6 @@ impl<'c> Serialize for PushArgs<'c> { } } -#[derive(Deserialize)] -pub struct DecodeQuery { - pub include_query: Option, -} - -#[derive(Deserialize)] -pub struct IncludeQuery { - pub include_query: Option, -} - -pub struct DecodeQueries(pub HashMap>); - -#[axum::async_trait] -impl FromRequestParts for DecodeQueries -where - S: Send + Sync, -{ - type Rejection = Response; - - async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { - Ok(DecodeQueries::from_uri(&parts.uri).unwrap_or_else(|| DecodeQueries(HashMap::new()))) - } -} - -impl DecodeQueries { - fn from_uri(uri: &Uri) -> Option { - let query = uri.query(); - if query.is_none() { - return None; - } - let query = query.unwrap(); - let include_query = serde_urlencoded::from_str::(query) - .map(|x| x.include_query) - .ok() - .flatten() - .unwrap_or_default(); - let parse_query_args = include_query - .split(",") - .map(|s| s.to_string()) - .collect::>(); - let mut args = HashMap::new(); - if !parse_query_args.is_empty() { - let queries = - serde_urlencoded::from_str::>(query).unwrap_or_default(); - parse_query_args.iter().for_each(|h| { - if let Some(v) = queries.get(h) { - args.insert(h.to_string(), to_raw_value(v)); - } - }); - } - Some(DecodeQueries(args)) - } -} - -// impl<'c> PushArgs<'c> { -// pub fn insert, V: Into>>(&mut self, k: K, v: V) { -// self.extra.insert(k.into(), v.into()); -// } -// } - -#[derive(Deserialize)] -pub struct RequestQuery { - pub raw: Option, - pub wrap_body: Option, - pub include_header: Option, -} - -fn restructure_cloudevents_metadata( - mut p: HashMap>, -) -> Result>, Error> { - let data = p - .remove("data") - .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); - let str = data.to_string(); - - let wrap_body = str.len() > 0 && str.chars().next().unwrap() != '{'; - - if wrap_body { - let args = serde_json::from_str::>>(&str) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? - .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); - let mut hm = HashMap::new(); - hm.insert("body".to_string(), args); - hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p)); - Ok(hm) - } else { - let mut hm = serde_json::from_str::>>>(&str) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))? - .unwrap_or_else(HashMap::new); - hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p)); - Ok(hm) - } -} - -impl PushArgsOwned { - async fn from_json( - mut extra: HashMap>, - use_raw: bool, - force_wrap_body: bool, - str: String, - ) -> Result { - if use_raw { - extra.insert("raw_string".to_string(), to_raw_value(&str)); - } - - let wrap_body = force_wrap_body || str.len() > 0 && str.chars().next().unwrap() != '{'; - - if wrap_body { - let args = serde_json::from_str::>>(&str) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())? - .unwrap_or_else(|| to_raw_value(&serde_json::Value::Null)); - let mut hm = HashMap::new(); - hm.insert("body".to_string(), args); - Ok(PushArgsOwned { extra: Some(extra), args: hm }) - } else { - let hm = serde_json::from_str::>>>(&str) - .map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())? - .unwrap_or_else(HashMap::new); - Ok(PushArgsOwned { extra: Some(extra), args: hm }) - } - } - - async fn from_ce_json( - mut extra: HashMap>, - use_raw: bool, - str: String, - ) -> Result { - if use_raw { - extra.insert("raw_string".to_string(), to_raw_value(&str)); - } - - let hm = serde_json::from_str::>>(&str).map_err(|e| { - Error::BadRequest(format!("invalid cloudevents+json: {}", e)).into_response() - })?; - let hm = restructure_cloudevents_metadata(hm).map_err(|e| e.into_response())?; - Ok(PushArgsOwned { extra: Some(extra), args: hm }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn test_cloudevents_json_payload() { - let r1 = r#" - { - "specversion" : "1.0", - "type" : "com.example.someevent", - "source" : "/mycontext", - "subject": null, - "id" : "C234-1234-1234", - "time" : "2018-04-05T17:31:00Z", - "comexampleextension1" : "value", - "comexampleothervalue" : 5, - "datacontenttype" : "application/json", - "data" : { - "appinfoA" : "abc", - "appinfoB" : 123, - "appinfoC" : true - } - } - "#; - let r2 = r#" - { - "specversion" : "1.0", - "type" : "com.example.someevent", - "source" : "/mycontext", - "subject": null, - "id" : "C234-1234-1234", - "time" : "2018-04-05T17:31:00Z", - "comexampleextension1" : "value", - "comexampleothervalue" : 5, - "datacontenttype" : "application/json", - "data" : 1.5 - } - "#; - let extra = HashMap::new(); - - let a1 = PushArgsOwned::from_ce_json(extra.clone(), false, r1.to_string()) - .await - .expect("Failed to parse the cloudevent"); - let a2 = PushArgsOwned::from_ce_json(extra.clone(), false, r2.to_string()) - .await - .expect("Failed to parse the cloudevent"); - - a1.args.get("WEBHOOK__METADATA__").expect( - "CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs", - ); - assert_eq!( - a2.args - .get("body") - .expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs") - .to_string(), - "1.5" - ); - } -} - -#[axum::async_trait] -impl FromRequest for PushArgsOwned -where - S: Send + Sync, -{ - type Rejection = Response; - - async fn from_request( - req: Request, - _state: &S, - ) -> Result { - let (content_type, mut extra, use_raw, wrap_body) = { - let headers_map = req.headers(); - let content_type_header = headers_map.get(CONTENT_TYPE); - let content_type = content_type_header.and_then(|value| value.to_str().ok()); - let uri = req.uri(); - let query = Query::::try_from_uri(uri).unwrap().0; - let mut extra = build_extra(&headers_map, query.include_header); - let query_decode = DecodeQueries::from_uri(uri); - if let Some(DecodeQueries(queries)) = query_decode { - extra.extend(queries); - } - let raw = query.raw.as_ref().is_some_and(|x| *x); - let wrap_body = query.wrap_body.as_ref().is_some_and(|x| *x); - (content_type, extra, raw, wrap_body) - }; - - let no_content_type = content_type.is_none(); - if no_content_type || content_type.unwrap().starts_with("application/json") { - let bytes = Bytes::from_request(req, _state) - .await - .map_err(IntoResponse::into_response)?; - if no_content_type && bytes.is_empty() { - if use_raw { - extra.insert("raw_string".to_string(), to_raw_value(&"".to_string())); - } - let mut args = HashMap::new(); - if wrap_body { - args.insert("body".to_string(), to_raw_value(&serde_json::json!({}))); - } - return Ok(PushArgsOwned { extra: Some(extra), args: args }); - } - let str = String::from_utf8(bytes.to_vec()) - .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; - - PushArgsOwned::from_json(extra, use_raw, wrap_body, str).await - } else if content_type - .unwrap() - .starts_with("application/cloudevents+json") - { - let bytes = Bytes::from_request(req, _state) - .await - .map_err(IntoResponse::into_response)?; - let str = String::from_utf8(bytes.to_vec()) - .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; - - PushArgsOwned::from_ce_json(extra, use_raw, str).await - } else if content_type - .unwrap() - .starts_with("application/cloudevents-batch+json") - { - Err( - Error::BadRequest(format!("Cloud events batching is not supported yet")) - .into_response(), - ) - } else if content_type.unwrap().starts_with("text/plain") { - let bytes = Bytes::from_request(req, _state) - .await - .map_err(IntoResponse::into_response)?; - let str = String::from_utf8(bytes.to_vec()) - .map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?; - extra.insert("raw_string".to_string(), to_raw_value(&str)); - Ok(PushArgsOwned { extra: Some(extra), args: HashMap::new() }) - } else if content_type - .unwrap() - .starts_with("application/x-www-form-urlencoded") - { - let bytes = Bytes::from_request(req, _state) - .await - .map_err(IntoResponse::into_response)?; - - if use_raw { - let raw_string = String::from_utf8(bytes.to_vec()).map_err(|e| { - Error::BadRequest(format!("invalid utf8: {}", e)).into_response() - })?; - extra.insert("raw_string".to_string(), to_raw_value(&raw_string)); - } - - let payload: HashMap> = serde_urlencoded::from_bytes(&bytes) - .map_err(|e| { - Error::BadRequest(format!("invalid urlencoded data: {}", e)).into_response() - })?; - let payload = payload - .into_iter() - .map(|(k, v)| (k, to_raw_value(&v))) - .collect::>(); - - return Ok(PushArgsOwned { extra: Some(extra), args: payload }); - } else { - Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response()) - } - } -} - -lazy_static::lazy_static! { - static ref INCLUDE_HEADERS: Vec = std::env::var("INCLUDE_HEADERS") - .ok().map(|x| x - .split(',') - .map(|s| s.to_string()) - .collect()).unwrap_or_default(); -} - -pub fn build_extra( - headers: &HeaderMap, - include_header: Option, -) -> HashMap> { - let mut args = HashMap::new(); - let whitelist = include_header - .map(|s| s.split(",").map(|s| s.to_string()).collect::>()) - .unwrap_or_default(); - - whitelist - .iter() - .chain(INCLUDE_HEADERS.iter()) - .for_each(|h| { - if let Some(v) = headers.get(h) { - args.insert( - h.to_string().to_lowercase().replace('-', "_"), - to_raw_value(&v.to_str().unwrap().to_string()), - ); - } - }); - args -} - impl PushArgsOwned { pub fn empty() -> Self { PushArgsOwned { extra: None, args: HashMap::new() } @@ -3229,6 +2879,26 @@ pub async fn push<'c, 'd>( None, ) } + JobPayload::AppScript { + id, // app_script(id). + path, + language, + cache_ttl, + } => ( + Some(id.0), + path, + None, + JobKind::AppScript, + None, + None, + Some(language), + None, + None, + None, + cache_ttl, + None, + None, + ), JobPayload::ScriptHub { path } => { if path == "hub/7771/slack" || path == "hub/7836/slack" { permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string(); @@ -3989,6 +3659,7 @@ pub async fn push<'c, 'd>( JobKind::DeploymentCallback => "jobs.run.deployment_callback", JobKind::FlowScript => "jobs.run.flow_script", JobKind::FlowNode => "jobs.run.flow_node", + JobKind::AppScript => "jobs.run.app_script", }; let audit_author = if format!("u/{user}") != permissioned_as && user != permissioned_as { @@ -4018,7 +3689,6 @@ pub async fn push<'c, 'd>( script_path.as_ref().map(|x| x.as_str()), Some(hm), ) - .instrument(tracing::info_span!("job_run", email = &email)) .await?; } diff --git a/backend/windmill-worker/Cargo.toml b/backend/windmill-worker/Cargo.toml index a8f065a4d2..8d9248b5b5 100644 --- a/backend/windmill-worker/Cargo.toml +++ b/backend/windmill-worker/Cargo.toml @@ -18,7 +18,12 @@ parquet = ["windmill-common/parquet", "dep:object_store"] flow_testing = [] cloud = [] sqlx = [] -deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", "dep:deno_ast", "dep:deno_tls"] +deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", + "dep:deno_ast", "dep:deno_tls", "dep:deno_permissions"] +otel = ["windmill-common/otel", "dep:opentelemetry"] +dind = ["dep:bollard"] +php = ["dep:windmill-parser-php"] +mysql = ["dep:mysql_async"] [dependencies] windmill-queue.workspace = true @@ -35,7 +40,7 @@ windmill-parser-py-imports.workspace = true windmill-parser-bash.workspace = true windmill-parser-sql.workspace = true windmill-parser-graphql.workspace = true -windmill-parser-php.workspace = true +windmill-parser-php = { workspace = true, optional = true } windmill-git-sync.workspace = true sqlx.workspace = true uuid.workspace = true @@ -69,9 +74,11 @@ deno_url = { workspace = true, optional = true } deno_core = { workspace = true, optional = true } deno_ast = { workspace = true, optional = true } deno_tls = { workspace = true, optional = true } +deno_permissions = { workspace = true, optional = true } + postgres-native-tls.workspace = true native-tls.workspace = true -mysql_async.workspace = true +mysql_async = { workspace = true, optional = true } base64.workspace = true gcp_auth = { workspace = true, optional = true } rust_decimal.workspace = true @@ -93,6 +100,9 @@ yaml-rust.workspace = true swc_ecma_parser.workspace = true backon.workspace = true +opentelemetry = { workspace = true, optional = true } +bollard = { workspace = true, optional = true } + [build-dependencies] deno_fetch = { workspace = true, optional = true } deno_webidl = { workspace = true, optional = true } @@ -103,4 +113,6 @@ deno_url = { workspace = true, optional = true } deno_core = { workspace = true, optional = true } deno_ast = { workspace = true, optional = true } deno_tls = { workspace = true, optional = true } +deno_permissions = { workspace = true, optional = true } + zstd.workspace = true diff --git a/backend/windmill-worker/build.rs b/backend/windmill-worker/build.rs index 6bbc3fd75f..e48ecddb35 100644 --- a/backend/windmill-worker/build.rs +++ b/backend/windmill-worker/build.rs @@ -25,7 +25,7 @@ impl FetchPermissions for PermissionsContainer { &mut self, _url: &deno_core::url::Url, _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { + ) -> Result<(), deno_permissions::PermissionCheckError> { unreachable!("snapshotting") } @@ -34,7 +34,7 @@ impl FetchPermissions for PermissionsContainer { &mut self, _p: &'a std::path::Path, _api_name: &str, - ) -> Result, deno_core::error::AnyError> { + ) -> Result, deno_permissions::PermissionCheckError> { unreachable!("snapshotting") } } @@ -53,7 +53,7 @@ impl NetPermissions for PermissionsContainer { &mut self, _p: &'a str, _api_name: &str, - ) -> Result { + ) -> Result { unreachable!("snapshotting") } @@ -61,7 +61,7 @@ impl NetPermissions for PermissionsContainer { &mut self, _p: &'a str, _api_name: &str, - ) -> Result { + ) -> Result { unreachable!("snapshotting") } @@ -69,7 +69,7 @@ impl NetPermissions for PermissionsContainer { &mut self, _host: &(T, Option), _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { + ) -> Result<(), deno_permissions::PermissionCheckError> { unreachable!("snapshotting") } @@ -77,7 +77,7 @@ impl NetPermissions for PermissionsContainer { &mut self, _: &'a Path, _: &str, - ) -> Result, deno_core::anyhow::Error> { + ) -> Result, deno_permissions::PermissionCheckError> { todo!() } } @@ -95,6 +95,7 @@ fn main() { println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); let exts = vec![ + // deno_telemetry::deno_telemetry::init_ops_and_esm(), deno_webidl::deno_webidl::init_ops_and_esm(), deno_url::deno_url::init_ops_and_esm(), deno_console::deno_console::init_ops_and_esm(), @@ -116,10 +117,10 @@ fn main() { deno_core::snapshot::CreateSnapshotOptions { cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"), startup_snapshot: None, + extension_transpiler: None, extensions: exts, with_runtime_cb: None, skip_op_registration: false, - extension_transpiler: None, }, None, ) diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 9bb67a1a6c..ba710925b7 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -115,8 +115,8 @@ async fn handle_ansible_python_deps( job_dir, worker_dir, &mut Some(occupancy_metrics), - true, - true, + false, + false, ) .await?; additional_python_paths.append(&mut venv_path); diff --git a/backend/windmill-worker/src/bash_executor.rs b/backend/windmill-worker/src/bash_executor.rs index 680831e990..b6279ccf97 100644 --- a/backend/windmill-worker/src/bash_executor.rs +++ b/backend/windmill-worker/src/bash_executor.rs @@ -1,14 +1,30 @@ use std::{collections::HashMap, fs, process::Stdio}; +#[cfg(feature = "dind")] +use bollard::container::{ + KillContainerOptions, RemoveContainerOptions, StatsOptions, StopContainerOptions, +}; +#[cfg(feature = "dind")] +use futures::{stream, StreamExt, TryStreamExt}; use regex::Regex; use serde_json::{json, value::RawValue}; use sqlx::types::Json; use tokio::process::Command; + +#[cfg(feature = "dind")] +use uuid::Uuid; use windmill_common::{ error::Error, jobs::QueuedJob, worker::{to_raw_value, write_file}, }; + +#[cfg(feature = "dind")] +use windmill_common::DB; + +#[cfg(feature = "dind")] +use windmill_common::error::to_anyhow; + use windmill_queue::{append_logs, CanceledBy}; lazy_static::lazy_static! { @@ -22,6 +38,9 @@ lazy_static::lazy_static! { static ref RE_POWERSHELL_IMPORTS: Regex = Regex::new(r#"^Import-Module\s+(?:-Name\s+)?"?([^-\s"]+)"?"#).unwrap(); } +#[cfg(feature = "dind")] +use crate::handle_child::run_future_with_polling_update_job_poller; + use crate::{ common::{ build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process, @@ -54,8 +73,14 @@ pub async fn handle_bash_job( worker_name: &str, envs: HashMap, occupancy_metrics: &mut OccupancyMetrics, + _killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, ) -> Result, Error> { - let logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string(); + let annotation = windmill_common::worker::BashAnnotations::parse(&content); + + let mut logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string(); + if annotation.docker { + logs1.push_str("docker mode\n"); + } append_logs(&job.id, &job.workspace_id, logs1, db).await; write_file(job_dir, "main.sh", &format!("set -e\n{content}"))?; @@ -193,6 +218,22 @@ exit $exit_status ) .await?; + #[cfg(feature = "dind")] + if annotation.docker { + return handle_docker_job( + job.id, + &job.workspace_id, + db, + job.timeout, + mem_peak, + canceled_by, + worker_name, + occupancy_metrics, + _killpill_rx, + ) + .await; + } + let result_json_path = format!("{job_dir}/result.json"); if let Ok(metadata) = tokio::fs::metadata(&result_json_path).await { if metadata.len() > 0 { @@ -214,6 +255,7 @@ exit $exit_status .await? .trim() .to_string(); + return Ok(to_raw_value(&json!(result))); } @@ -222,6 +264,190 @@ exit $exit_status ))) } +#[cfg(feature = "dind")] +async fn handle_docker_job( + job_id: Uuid, + workspace_id: &str, + db: &DB, + job_timeout: Option, + mem_peak: &mut i32, + canceled_by: &mut Option, + worker_name: &str, + occupancy_metrics: &mut OccupancyMetrics, + killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, +) -> Result, Error> { + let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow)?; + + let container_id = job_id.to_string(); + let inspected = client.inspect_container(&container_id, None).await; + + if inspected.is_err() || inspected.unwrap().state.is_none() { + return Ok(to_raw_value(&format!( + "Container not found at {job_id}, you must use --name and not --rm it" + ))); + } + + let wait_f = async { + let wait = client + .wait_container::(&container_id, None) + .try_collect::>() + .await + .map_err(|e| { + tracing::error!("Error waiting for container: {:?}", e); + anyhow::anyhow!("Error waiting for container") + })?; + let waited = wait.first().map(|x| x.status_code); + Ok(waited) + }; + + let ncontainer_id = container_id.to_string(); + let w_id = workspace_id.to_string(); + let j_id = job_id.clone(); + let db2 = db.clone(); + let (tx, mut rx) = tokio::sync::broadcast::channel::<()>(1); + + let mut killpill_rx = killpill_rx.resubscribe(); + let logs = tokio::spawn(async move { + let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow); + if let Ok(client) = client { + let mut log_stream = client.logs( + &ncontainer_id, + Some(bollard::container::LogsOptions { + follow: true, + stdout: true, + stderr: true, + tail: "all", + ..Default::default() + }), + ); + loop { + tokio::select! { + log = log_stream.next() => { + match log { + Some(Ok(log)) => { + append_logs(&j_id, w_id.clone(), log.to_string(), db2.clone()).await; + } + Some(Err(e)) => { + tracing::error!("Error getting logs: {:?}", e); + } + _ => { + tracing::error!("End of stream"); + return + } + }; + }, + _ = killpill_rx.recv() => { + tracing::error!("killing container after receving killpill"); + if let Err(e) = client + .stop_container(&ncontainer_id, Some(StopContainerOptions { t: 3 })) + .await + { + tracing::error!("Error stopping container: {:?}", e); + } + return + }, + _ = rx.recv() => { + return + } + } + } + } + }); + + let mem_client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow); + let ncontainer_id = container_id.clone(); + let result = run_future_with_polling_update_job_poller( + job_id, + job_timeout, + db, + mem_peak, + canceled_by, + wait_f, + worker_name, + workspace_id, + &mut Some(occupancy_metrics), + Box::pin(match mem_client { + Ok(client) => client + .stats( + &ncontainer_id, + Some(StatsOptions { stream: true, one_shot: false }), + ) + .map(|x| { + x.map(|x| x.memory_stats.usage.map(|m| m / 1024).unwrap_or_default() as i32) + .unwrap_or_default() + }) + .boxed(), + _ => stream::once(async { 0 }).boxed(), + }), + ) + .await; + + if let Err(e) = result { + if !logs.is_finished() { + let _ = tx.send(()); + let _ = logs.await; + } + if container_is_alive(&client, &container_id).await { + kill_container(&client, &container_id, "SIGINT").await; + if container_is_alive(&client, &container_id).await { + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + if let Err(e) = client + .stop_container(&container_id, Some(StopContainerOptions { t: 3 })) + .await + { + tracing::error!("Error stopping container: {:?}", e); + } + } + } + + return Err(e); + } + + if let Err(e) = client + .remove_container( + &container_id, + Some(RemoveContainerOptions { force: true, ..Default::default() }), + ) + .await + { + tracing::error!("Error removing container: {:?}", e); + } + + let result = result.unwrap(); + + return Ok(to_raw_value(&json!(format!( + "Docker exit status: {}", + result + .map(|x| x.to_string()) + .unwrap_or_else(|| "none".to_string()) + )))); +} + +#[cfg(feature = "dind")] +async fn kill_container(client: &bollard::Docker, container_id: &str, signal: &str) { + if let Err(e) = client + .kill_container(&container_id, Some(KillContainerOptions { signal })) + .await + { + tracing::error!("Error killing container with signal {signal}: {:?}", e); + } +} + +#[cfg(feature = "dind")] +async fn container_is_alive(client: &bollard::Docker, container_id: &str) -> bool { + let inspect = client.inspect_container(container_id, None).await; + if let Ok(inspect) = inspect { + let r = inspect + .state + .map(|x| x.running.unwrap_or_default()) + .unwrap_or_default(); + tracing::error!("Container {container_id} is alive: {r}"); + r + } else { + false + } +} + fn raw_to_string(x: &str) -> String { match serde_json::from_str::(x) { Ok(serde_json::Value::String(x)) => x, diff --git a/backend/windmill-worker/src/bigquery_executor.rs b/backend/windmill-worker/src/bigquery_executor.rs index 2438392680..30c0f1d575 100644 --- a/backend/windmill-worker/src/bigquery_executor.rs +++ b/backend/windmill-worker/src/bigquery_executor.rs @@ -366,6 +366,7 @@ pub async fn do_bigquery( worker_name, &job.workspace_id, &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await?; diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index b12406e512..ed268ec5ad 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -1,11 +1,12 @@ use async_recursion::async_recursion; use itertools::Itertools; - +use lazy_static::lazy_static; use regex::Regex; use serde::{Deserialize, Serialize}; use serde_json::value::RawValue; use serde_json::{json, Value}; +use sha2::Digest; use sqlx::types::Json; use sqlx::{Pool, Postgres}; use tokio::process::Command; @@ -21,8 +22,10 @@ use windmill_common::worker::{ to_raw_value, write_file, CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG, }; use windmill_common::{ + cache::Cache, error::{self, Error}, jobs::QueuedJob, + scripts::ScriptHash, variables::ContextualVariable, }; @@ -32,6 +35,7 @@ use std::path::Path; use std::{ collections::{hash_map::DefaultHasher, HashMap}, hash::{Hash, Hasher}, + sync::Arc, time::Duration, }; @@ -633,40 +637,59 @@ pub async fn resolve_job_timeout( } } -pub async fn hash_args( +async fn hash_args( _db: &DB, _client: &AuthedClient, _workspace_id: &str, - _job_id: &Uuid, - v: &Option>>>, -) -> String { - if let Some(vs) = v { - let mut dh = DefaultHasher::new(); - let hm = &vs.0; + v: &Option>>>, + hasher: &mut sha2::Sha256, +) { + if let Some(Json(hm)) = v { for k in hm.keys().sorted() { - k.hash(&mut dh); + hasher.update(k.as_bytes()); let arg_value = hm.get(k).unwrap(); #[cfg(feature = "parquet")] let (_, arg_additions) = arg_value_hash_additions(_db, _client, _workspace_id, hm.get(k).unwrap()).await; - arg_value.get().hash(&mut dh); + hasher.update(arg_value.get().as_bytes()); #[cfg(feature = "parquet")] for (_, arg_addition) in arg_additions { - arg_addition.hash(&mut dh); + hasher.update(arg_addition.as_bytes()); } } - hex::encode(dh.finish().to_be_bytes()) - } else { - "empty_args".to_string() } } +pub async fn cached_result_path( + db: &DB, + client: &AuthedClient, + job: &QueuedJob, + raw_code: Option<&String>, + raw_lock: Option<&String>, + raw_flow: Option<&Json>>, +) -> String { + let mut hasher = sha2::Sha256::new(); + hasher.update(&[job.job_kind as u8]); + if let Some(ScriptHash(hash)) = job.script_hash { + hasher.update(&hash.to_le_bytes()) + } else if let None = job.script_hash { + job.script_path + .as_ref() + .inspect(|x| hasher.update(x.as_bytes())); + raw_code.inspect(|x| hasher.update(x)); + raw_lock.inspect(|x| hasher.update(x)); + raw_flow.inspect(|x| hasher.update(x.get())); + } + hash_args(db, client, &job.workspace_id, &job.args, &mut hasher).await; + format!("g/results/{:032x}", hasher.finalize()) +} + #[cfg(feature = "parquet")] async fn get_workspace_s3_resource_path( db: &DB, client: &AuthedClient, workspace_id: &str, - storage: Option, + storage: Option<&String>, ) -> windmill_common::error::Result> { use windmill_common::{ job_s3_helpers_ee::get_s3_resource_internal, s3_helpers::StorageResourceType, @@ -750,7 +773,7 @@ async fn arg_value_hash_additions( let mut storage = None; if let Ok(s3_object) = parsed_value { let s3_resource_opt = - get_workspace_s3_resource_path(db, client, workspace_id, s3_object.storage.clone()) + get_workspace_s3_resource_path(db, client, workspace_id, s3_object.storage.as_ref()) .await; storage = s3_object.storage.clone(); @@ -769,68 +792,93 @@ struct CachedResource { expire: i64, #[serde(skip_serializing_if = "Option::is_none")] s3_etags: Option>, - value: Box, + value: Arc>, storage: Option, } +impl CachedResource { + fn expired(&self) -> bool { + self.expire <= chrono::Utc::now().timestamp() + } +} + +lazy_static! { + /// In-memory cache for resources. + static ref CACHED_RESULTS: Cache> = Cache::new(1000); +} + pub async fn get_cached_resource_value_if_valid( _db: &DB, client: &AuthedClient, _job_id: &Uuid, _workspace_id: &str, cached_res_path: &str, -) -> Option> { - let resource_opt = client - .get_resource_value::(cached_res_path) +) -> Option>> { + let resource = match CACHED_RESULTS + .get_value_or_guard_async(cached_res_path) .await - .ok(); - if let Some(cached_resource) = resource_opt { - if cached_resource.expire <= chrono::Utc::now().timestamp() { - // cache expired + { + // check for cache expiration. + Ok(resource) if resource.expired() => { + let _ = CACHED_RESULTS.remove(cached_res_path); return None; } - #[cfg(feature = "parquet")] + // resource is in cache and not expired, return it. + Ok(resource) => resource, + // resource is not in cache, fetch it from the database. + Err(entry) => match client + .get_resource_value::(cached_res_path) + .await + .ok() { - let s3_etags = cached_resource.s3_etags.unwrap_or_default(); - let object_store_resource_opt: Option = if s3_etags.is_empty() { - None - } else { - get_workspace_s3_resource_path( - _db, - &client, - _workspace_id, - cached_resource.storage.clone(), - ) + None => return None, + // check for cache expiration. + Some(resource) if resource.expired() => return None, + Some(resource) => { + let resource = Arc::new(resource); + let _ = entry.insert(resource.clone()); + resource + } + }, + }; + + #[cfg(feature = "parquet")] + { + let empty_etags = HashMap::new(); + let s3_etags = resource.s3_etags.as_ref().unwrap_or(&empty_etags); + let object_store_resource_opt: Option = if s3_etags.is_empty() { + None + } else { + get_workspace_s3_resource_path(_db, &client, _workspace_id, resource.storage.as_ref()) .await .ok() .flatten() - }; + }; - if !s3_etags.is_empty() && object_store_resource_opt.is_none() { - tracing::warn!("Cached result references s3 files that are not retrievable anymore because the workspace S3 resource can't be fetched. Cache will be invalidated"); - return None; - } - for (s3_file_key, s3_file_etag) in s3_etags { - if let Some(mut object_store_resource) = object_store_resource_opt.clone() { - let etag = get_etag_or_empty( - &mut object_store_resource, - S3Object { - s3: s3_file_key.clone(), - storage: cached_resource.storage.clone(), - filename: None, - }, - ) - .await; - if etag.is_none() || etag.clone().unwrap() != s3_file_etag { - tracing::warn!("S3 file etag for '{}' has changed. Value from cache is {:?} while current value from S3 is {:?}. Cache will be invalidated", s3_file_key.clone(), s3_file_etag, etag); - return None; - } + if !s3_etags.is_empty() && object_store_resource_opt.is_none() { + tracing::warn!("Cached result references s3 files that are not retrievable anymore because the workspace S3 resource can't be fetched. Cache will be invalidated"); + return None; + } + for (s3_file_key, s3_file_etag) in s3_etags { + if let Some(object_store_resource) = object_store_resource_opt.as_ref() { + let etag = get_etag_or_empty( + object_store_resource, + S3Object { + s3: s3_file_key.clone(), + storage: resource.storage.clone(), + filename: None, + }, + ) + .await; + if etag.as_ref() != Some(s3_file_etag) { + tracing::warn!("S3 file etag for '{}' has changed. Value from cache is {:?} while current value from S3 is {:?}. Cache will be invalidated", s3_file_key.clone(), s3_file_etag, etag); + return None; } } } - return Some(cached_resource.value); } - return None; + + Some(resource.value.clone()) } pub async fn save_in_cache( @@ -838,13 +886,13 @@ pub async fn save_in_cache( _client: &AuthedClient, job: &QueuedJob, cached_path: String, - r: &Box, + r: Arc>, ) { let expire = chrono::Utc::now().timestamp() + job.cache_ttl.unwrap() as i64; #[cfg(feature = "parquet")] let (storage, s3_etags) = - arg_value_hash_additions(db, _client, job.workspace_id.as_str(), r).await; + arg_value_hash_additions(db, _client, job.workspace_id.as_str(), &r).await; #[cfg(feature = "parquet")] let s3_etags = if s3_etags.is_empty() { @@ -856,17 +904,17 @@ pub async fn save_in_cache( #[cfg(not(feature = "parquet"))] let (storage, s3_etags) = (None, None); - let store_cache_resource = CachedResource { expire, s3_etags, value: r.clone(), storage }; - let raw_json = sqlx::types::Json(store_cache_resource); + let store_cache_resource = CachedResource { expire, s3_etags, value: r, storage }; + let raw_json = Json(&store_cache_resource); if let Err(e) = sqlx::query!( "INSERT INTO resource - (workspace_id, path, value, resource_type, created_by, edited_at) - VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path) - DO UPDATE SET value = $3, edited_at = now()", + (workspace_id, path, value, resource_type, created_by, edited_at) + VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path) + DO UPDATE SET value = $3, edited_at = now()", job.workspace_id, - cached_path, - raw_json as sqlx::types::Json, + &cached_path, + raw_json as Json<&CachedResource>, "cache", job.created_by ) @@ -875,6 +923,9 @@ pub async fn save_in_cache( { tracing::error!("Error creating cache resource {e:#}") } + + // Cache result in-memory. + CACHED_RESULTS.insert(cached_path, Arc::new(store_cache_resource)); } fn tentatively_improve_error(err: Error, executable: &str) -> Error { @@ -884,11 +935,11 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error { #[cfg(windows)] let err_msg = "program not found"; - if err - .to_string() - .contains(&err_msg) - { - return Error::InternalErr(format!("Executable {executable} not found on worker. PATH: {}", *PATH_ENV)); + if err.to_string().contains(&err_msg) { + return Error::InternalErr(format!( + "Executable {executable} not found on worker. PATH: {}", + *PATH_ENV + )); } return err; } diff --git a/backend/windmill-worker/src/dedicated_worker.rs b/backend/windmill-worker/src/dedicated_worker.rs index c16413c573..ad1333c28d 100644 --- a/backend/windmill-worker/src/dedicated_worker.rs +++ b/backend/windmill-worker/src/dedicated_worker.rs @@ -16,8 +16,7 @@ use windmill_common::error::Error; use windmill_common::flows::FlowValue; use windmill_common::worker::WORKER_CONFIG; use windmill_common::{ - cache, - error, + cache, error, flows::{FlowModule, FlowModuleValue}, jobs::QueuedJob, scripts::{ScriptHash, ScriptLang}, @@ -416,17 +415,18 @@ async fn spawn_dedicated_workers_for_flow( job_completed_tx, Some(module.id.clone()), ) - .await + .await { workers.push(dedi_w); } - }, + } Err(err) => tracing::error!( "failed to get script for module: {:?}, err: {:?}", - module, err - ) + module, + err + ), } - }, + } FlowModuleValue::Flow { .. } => (), FlowModuleValue::Identity => (), } diff --git a/backend/windmill-worker/src/graphql_executor.rs b/backend/windmill-worker/src/graphql_executor.rs index 8fb8cf13b9..67bb6a2c7b 100644 --- a/backend/windmill-worker/src/graphql_executor.rs +++ b/backend/windmill-worker/src/graphql_executor.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use anyhow::anyhow; -use futures::TryStreamExt; +use futures::{stream, TryStreamExt}; use serde_json::{json, value::RawValue}; use sqlx::types::Json; use windmill_common::jobs::QueuedJob; @@ -154,6 +154,7 @@ pub async fn do_graphql( worker_name, &job.workspace_id, &mut Some(occupation_metrics), + Box::pin(stream::once(async { 0 })), ) .await?; diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 4d09c36abc..f2fead39c9 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -87,7 +87,7 @@ async fn kill_process_tree(pid: Option) -> Result<(), String> { /// - update the `last_line` and `logs` strings with the program output /// - update "queue"."last_ping" every five seconds /// - kill process if we exceed timeout or "queue"."canceled" is set -#[tracing::instrument(level = "trace", skip_all)] +#[tracing::instrument(name="run_subprocess", level = "info", skip_all, fields(otel.name = %child_name))] pub async fn handle_child( job_id: &Uuid, db: &Pool, @@ -126,7 +126,7 @@ pub async fn handle_child( let (tx, rx) = broadcast::channel::<()>(3); let mut rx2 = tx.subscribe(); - let output = child_joined_output_stream(&mut child); + let output = child_joined_output_stream(&mut child, job_id.clone()); let job_id = job_id.clone(); @@ -137,7 +137,9 @@ pub async fn handle_child( db, mem_peak, canceled_by_ref, - || get_mem_peak(pid, nsjail), + Box::pin(stream::unfold((), move |_| async move { + Some((get_mem_peak(pid, nsjail).await, ())) + })), worker, w_id, rx, @@ -435,8 +437,6 @@ pub async fn handle_child( } } - - async fn get_mem_peak(pid: Option, nsjail: bool) -> i32 { if pid.is_none() { return -1; @@ -488,7 +488,7 @@ async fn get_mem_peak(pid: Option, nsjail: bool) -> i32 { } } -pub async fn run_future_with_polling_update_job_poller( +pub async fn run_future_with_polling_update_job_poller( job_id: Uuid, timeout: Option, db: &DB, @@ -498,9 +498,11 @@ pub async fn run_future_with_polling_update_job_poller( worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + get_mem: S, ) -> error::Result where Fut: Future>, + S: stream::Stream + Unpin, { let (tx, rx) = broadcast::channel::<()>(3); @@ -509,7 +511,7 @@ where db, mem_peak, canceled_by_ref, - || async { 0 }, + get_mem, worker_name, w_id, rx, @@ -550,20 +552,19 @@ pub enum UpdateJobPollingExit { AlreadyCompleted, } -pub async fn update_job_poller( +pub async fn update_job_poller( job_id: Uuid, db: &DB, mem_peak: &mut i32, canceled_by_ref: &mut Option, - get_mem: F, + mut get_mem: S, worker_name: &str, w_id: &str, mut rx: broadcast::Receiver<()>, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> UpdateJobPollingExit where - F: Fn() -> Fut, - Fut: Future, + S: stream::Stream + Unpin, { let update_job_interval = Duration::from_millis(500); @@ -608,7 +609,7 @@ where .expect("update worker ping"); } } - let current_mem = get_mem().await; + let current_mem = get_mem.next().await.unwrap_or(0); if current_mem > *mem_peak { *mem_peak = current_mem } @@ -679,6 +680,7 @@ where /// builds a stream joining both stdout and stderr each read line by line fn child_joined_output_stream( child: &mut Child, + job_id: Uuid, ) -> impl stream::FusedStream> { let stderr = child .stderr @@ -692,24 +694,24 @@ fn child_joined_output_stream( let stdout = BufReader::new(stdout).lines(); let stderr = BufReader::new(stderr).lines(); - stream::select(lines_to_stream(stderr, true), lines_to_stream(stdout, false)) + stream::select( + lines_to_stream(stderr, true, job_id.clone()), + lines_to_stream(stdout, false, job_id), + ) } pub fn lines_to_stream( mut lines: tokio::io::Lines, stderr: bool, + job_id: Uuid, ) -> impl futures::Stream> { stream::poll_fn(move |cx| { std::pin::Pin::new(&mut lines) .poll_next_line(cx) - .map(|result| { - process_streaming_log_lines(result, stderr) - }) + .map(|result| process_streaming_log_lines(result, stderr, &job_id)) }) } - - pub fn process_status(status: ExitStatus) -> error::Result<()> { if status.success() { Ok(()) diff --git a/backend/windmill-worker/src/job_logger.rs b/backend/windmill-worker/src/job_logger.rs index f92ee16536..acf42e1f4d 100644 --- a/backend/windmill-worker/src/job_logger.rs +++ b/backend/windmill-worker/src/job_logger.rs @@ -15,7 +15,6 @@ use crate::job_logger_ee::default_disk_log_storage; #[cfg(all(feature = "enterprise", feature = "parquet"))] use crate::job_logger_ee::s3_storage; - pub enum CompactLogs { #[cfg(not(all(feature = "enterprise", feature = "parquet")))] NotEE, @@ -25,8 +24,6 @@ pub enum CompactLogs { S3, } - - pub(crate) async fn append_job_logs( job_id: Uuid, w_id: String, @@ -70,7 +67,7 @@ pub fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) { if *NO_LOGS_AT_ALL { return; } - + let src_str; let src = { src_str = RE_00.replace_all(src, ""); diff --git a/backend/windmill-worker/src/job_logger_ee.rs b/backend/windmill-worker/src/job_logger_ee.rs index 5414ccd496..15bd4f83b2 100644 --- a/backend/windmill-worker/src/job_logger_ee.rs +++ b/backend/windmill-worker/src/job_logger_ee.rs @@ -12,11 +12,11 @@ pub(crate) async fn s3_storage( _job_id: Uuid, _w_id: &String, _db: &sqlx::Pool, - _logs: &String, - _total_size: &Arc, + _logs: String, + _total_size: Arc, _worker_name: &String, ) { - tracing::info!("Logs length of {job_id} has exceeded a threshold. Implementation to store excess on s3 in not OSS"); + tracing::info!("Logs length of {_job_id} has exceeded a threshold. Implementation to store excess on s3 in not OSS"); } pub(crate) async fn default_disk_log_storage( @@ -34,6 +34,7 @@ pub(crate) async fn default_disk_log_storage( pub(crate) fn process_streaming_log_lines( r: Result, io::Error>, _stderr: bool, + _job_id: &Uuid, ) -> Option> { r.transpose() } diff --git a/backend/windmill-worker/src/js_eval.rs b/backend/windmill-worker/src/js_eval.rs index b4b84e2e2b..482bfea3ed 100644 --- a/backend/windmill-worker/src/js_eval.rs +++ b/backend/windmill-worker/src/js_eval.rs @@ -7,14 +7,7 @@ */ #[cfg(feature = "deno_core")] -use std::{ - borrow::Cow, - cell::RefCell, - env, - io::{self, BufReader}, - path::PathBuf, - rc::Rc, -}; +use std::{borrow::Cow, cell::RefCell, env, path::PathBuf, rc::Rc}; use std::{collections::HashMap, sync::Arc}; @@ -22,7 +15,6 @@ use std::{collections::HashMap, sync::Arc}; use deno_ast::ParseParams; #[cfg(feature = "deno_core")] use deno_core::{ - error::AnyError, op2, serde_v8, url, v8::{self, IsolateHandle}, Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions, @@ -31,8 +23,7 @@ use deno_core::{ use deno_fetch::FetchPermissions; #[cfg(feature = "deno_core")] use deno_net::NetPermissions; -#[cfg(feature = "deno_core")] -use deno_tls::{rustls::RootCertStore, rustls_pemfile}; + #[cfg(feature = "deno_core")] use deno_web::{BlobStore, TimersPermission}; #[cfg(feature = "deno_core")] @@ -68,35 +59,35 @@ pub struct IdContext { pub previous_id: String, } -#[cfg(feature = "deno_core")] -pub struct ContainerRootCertStoreProvider { - root_cert_store: RootCertStore, -} +// #[cfg(feature = "deno_core")] +// pub struct ContainerRootCertStoreProvider { +// root_cert_store: RootCertStore, +// } -#[cfg(feature = "deno_core")] -impl ContainerRootCertStoreProvider { - fn new() -> ContainerRootCertStoreProvider { - return ContainerRootCertStoreProvider { - root_cert_store: deno_tls::create_default_root_cert_store(), - }; - } +// #[cfg(feature = "deno_core")] +// impl ContainerRootCertStoreProvider { +// fn new() -> ContainerRootCertStoreProvider { +// return ContainerRootCertStoreProvider { +// root_cert_store: deno_tls::create_default_root_cert_store(), +// }; +// } - fn add_certificate(&mut self, cert_path: String) -> io::Result<()> { - let cert_file = std::fs::File::open(cert_path)?; - let mut reader = BufReader::new(cert_file); - let pem_file = rustls_pemfile::certs(&mut reader).collect::, _>>()?; +// fn add_certificate(&mut self, cert_path: String) -> io::Result<()> { +// let cert_file = std::fs::File::open(cert_path)?; +// let mut reader = BufReader::new(cert_file); +// let pem_file = rustls_pemfile::certs(&mut reader).collect::, _>>()?; - self.root_cert_store.add_parsable_certificates(pem_file); - Ok(()) - } -} +// self.root_cert_store.add_parsable_certificates(pem_file); +// Ok(()) +// } +// } -#[cfg(feature = "deno_core")] -impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider { - fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> { - Ok(&self.root_cert_store) - } -} +// #[cfg(feature = "deno_core")] +// impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider { +// fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> { +// Ok(&self.root_cert_store) +// } +// } #[cfg(feature = "deno_core")] pub struct PermissionsContainer; @@ -108,7 +99,7 @@ impl FetchPermissions for PermissionsContainer { &mut self, _url: &deno_core::url::Url, _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { + ) -> Result<(), deno_permissions::PermissionCheckError> { Ok(()) } @@ -117,7 +108,7 @@ impl FetchPermissions for PermissionsContainer { &mut self, p: &'a std::path::Path, _api_name: &str, - ) -> Result, anyhow::Error> { + ) -> Result, deno_permissions::PermissionCheckError> { Ok(Cow::Borrowed(p)) } } @@ -136,7 +127,7 @@ impl NetPermissions for PermissionsContainer { &mut self, p: &'a str, _api_name: &str, - ) -> Result { + ) -> Result { Ok(PathBuf::from(p)) } @@ -144,7 +135,7 @@ impl NetPermissions for PermissionsContainer { &mut self, p: &'a str, _api_name: &str, - ) -> Result { + ) -> Result { Ok(PathBuf::from(p)) } @@ -152,7 +143,7 @@ impl NetPermissions for PermissionsContainer { &mut self, _host: &(T, Option), _api_name: &str, - ) -> Result<(), deno_core::error::AnyError> { + ) -> Result<(), deno_permissions::PermissionCheckError> { Ok(()) } @@ -160,7 +151,7 @@ impl NetPermissions for PermissionsContainer { &mut self, p: &'a std::path::Path, _api_name: &str, - ) -> Result, AnyError> { + ) -> Result, deno_permissions::PermissionCheckError> { Ok(Cow::Borrowed(p)) } } @@ -669,9 +660,12 @@ pub fn transpile_ts(expr: String) -> anyhow::Result { text: deno_core::ModuleCodeString::from(expr).into(), })?; Ok(parsed - .transpile(&Default::default(), &Default::default())? + .transpile( + &Default::default(), + &Default::default(), + &Default::default(), + )? .into_source() - .into_string()? .text) } @@ -814,13 +808,7 @@ pub async fn eval_fetch_timeout( let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() }; let fetch_options = deno_fetch::Options { - root_cert_store_provider: if let Some(cert_path) = env::var("DENO_CERT").ok() { - let mut cert_store_provider = ContainerRootCertStoreProvider::new(); - cert_store_provider.add_certificate(cert_path)?; - Some(Arc::new(cert_store_provider)) - } else { - None - }, + root_cert_store_provider: None, user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()), proxy: ann.proxy.map(|x| deno_tls::Proxy { url: x.0, @@ -930,6 +918,7 @@ pub async fn eval_fetch_timeout( worker_name, w_id, &mut Some(occupation_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await .map_err(|e| { diff --git a/backend/windmill-worker/src/lib.rs b/backend/windmill-worker/src/lib.rs index 3a2efd75d8..c68f77917f 100644 --- a/backend/windmill-worker/src/lib.rs +++ b/backend/windmill-worker/src/lib.rs @@ -20,6 +20,7 @@ mod graphql_executor; mod handle_child; mod job_logger; mod js_eval; +#[cfg(feature = "mysql")] mod mysql_executor; mod pg_executor; mod php_executor; diff --git a/backend/windmill-worker/src/mssql_executor.rs b/backend/windmill-worker/src/mssql_executor.rs index 7c4724418c..b97560e225 100644 --- a/backend/windmill-worker/src/mssql_executor.rs +++ b/backend/windmill-worker/src/mssql_executor.rs @@ -160,6 +160,7 @@ pub async fn do_mssql( worker_name, &job.workspace_id, &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await?; diff --git a/backend/windmill-worker/src/mysql_executor.rs b/backend/windmill-worker/src/mysql_executor.rs index 3ed6a9bf34..b98ec365ee 100644 --- a/backend/windmill-worker/src/mysql_executor.rs +++ b/backend/windmill-worker/src/mysql_executor.rs @@ -298,6 +298,7 @@ pub async fn do_mysql( worker_name, &job.workspace_id, &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await?; diff --git a/backend/windmill-worker/src/pg_executor.rs b/backend/windmill-worker/src/pg_executor.rs index 57d7f48905..2ad6eb4038 100644 --- a/backend/windmill-worker/src/pg_executor.rs +++ b/backend/windmill-worker/src/pg_executor.rs @@ -355,6 +355,7 @@ pub async fn do_postgresql( worker_name, &job.workspace_id, &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await?; diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 345d8e37b7..25000e020c 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -192,9 +192,13 @@ pub async fn handle_php_job( let main_override = get_main_override(job.args.as_ref()); let write_wrapper_f = async { + #[cfg(feature = "php")] let args = windmill_parser_php::parse_php_signature(inner_content, main_override.clone())?.args; + #[cfg(not(feature = "php"))] + let args: Vec = vec![]; + let args_to_include = args .iter() .filter(|x| { diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 3d1ba79def..3c31fb1913 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -3,7 +3,7 @@ use std::{ fs, path::Path, process::Stdio, - sync::Arc + sync::Arc, }; use anyhow::anyhow; @@ -51,11 +51,11 @@ lazy_static::lazy_static! { static ref PIP_TRUSTED_HOST: Option = std::env::var("PIP_TRUSTED_HOST").ok(); static ref PIP_INDEX_CERT: Option = std::env::var("PIP_INDEX_CERT").ok(); - static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") + pub static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") .ok().map(|flag| flag == "true").unwrap_or(false); /// Use pip install - static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") + pub static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL") .ok().map(|flag| flag == "true").unwrap_or(false); @@ -318,7 +318,7 @@ pub async fn uv_pip_compile( if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { args.extend(["--cert", cert_path]); } - tracing::debug!("uv args: {:?}", args); + tracing::error!("uv args: {:?}", args); #[cfg(windows)] let uv_cmd = "uv"; @@ -329,7 +329,10 @@ pub async fn uv_pip_compile( let mut child_cmd = Command::new(uv_cmd); child_cmd .current_dir(job_dir) - .args(args) + .env_clear() + .env("HOME", HOME_ENV.to_string()) + .env("PATH", PATH_ENV.to_string()) + .args(&args) .stdout(Stdio::piped()) .stderr(Stdio::piped()); let child_process = start_child_process(child_cmd, uv_cmd).await?; @@ -350,7 +353,12 @@ pub async fn uv_pip_compile( occupancy_metrics, ) .await - .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + .map_err(|e| { + Error::ExecutionErr(format!( + "Lock file generation failed.\n\ncommand: {uv_cmd} {}\n\n{e:?}", + args.join(" ") + )) + })?; } let path_lock = format!("{job_dir}/requirements.txt"); @@ -423,12 +431,12 @@ async fn postinstall( .ok_or(anyhow::anyhow!("Cannot convert OsString to String"))? .to_owned(); - if name == "bin" || name.contains("dist-info") { + if name == "bin" || name == "__pycache__" || name.contains("dist-info") { continue; } if let Some(existing_paths) = lookup_table.get_mut(&name) { - tracing::info!( + tracing::debug!( "Found existing package name: {:?} in {}", entry.file_name(), path @@ -479,18 +487,22 @@ fn copy_dir_recursively(src: &Path, dst: &Path) -> windmill_common::error::Resul fs::create_dir_all(dst)?; } + tracing::debug!("Copying recursively from {:?} to {:?}", src, dst); + for entry in fs::read_dir(src)? { let entry = entry?; let src_path = entry.path(); let dst_path = dst.join(entry.file_name()); - if src_path.is_dir() { + if src_path.is_dir() && !src_path.is_symlink() { copy_dir_recursively(&src_path, &dst_path)?; } else { fs::copy(&src_path, &dst_path)?; } } + tracing::debug!("Finished copying recursively from {:?} to {:?}", src, dst); + Ok(()) } @@ -529,10 +541,13 @@ pub async fn handle_python_job( ) .await?; + tracing::debug!("Finished handling python dependencies"); + if !PythonAnnotations::parse(inner_content).no_postinstall { if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, db).await { tracing::error!("Postinstall stage has failed. Reason: {e}"); } + tracing::debug!("Finished deps postinstall stage"); } append_logs( @@ -563,9 +578,12 @@ pub async fn handle_python_job( ) .await?; + tracing::debug!("Finished preparing wrapper"); + let apply_preprocessor = pre_spread.is_some(); create_args_and_out_file(&client, job, job_dir, db).await?; + tracing::debug!("Finished preparing wrapper"); let preprocessor = if let Some(pre_spread) = pre_spread { format!( @@ -664,6 +682,8 @@ except BaseException as e: ); write_file(job_dir, "wrapper.py", &wrapper_content)?; + tracing::debug!("Finished writing wrapper"); + let client = client.get_authed().await; let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?; let additional_python_paths_folders = additional_python_paths.iter().join(":"); @@ -749,7 +769,8 @@ mount {{ .stdout(Stdio::piped()) .stderr(Stdio::piped()); - #[cfg(windows)] { + #[cfg(windows)] + { python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str()); python_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str()); } @@ -1093,7 +1114,7 @@ async fn handle_python_deps( let mut venv_path = handle_python_reqs( requirements .split("\n") - .filter(|x| !x.starts_with("--")) + .filter(|x| !x.starts_with("--") && !x.trim().is_empty()) .collect(), job_id, w_id, @@ -1239,29 +1260,28 @@ async fn spawn_uv_install( #[cfg(unix)] { if no_uv_install { - let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); - flock_cmd - .env_clear() - .envs(PROXY_ENVS.clone()) - .envs(envs) - .args([ - "-x", - &format!( - "{}/{}-{}.lock", - LOCK_CACHE_DIR, - if no_uv_install { "pip" } else { "py311" }, - fssafe_req - ), - "--command", - &command_args.join(" "), - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - start_child_process(flock_cmd, FLOCK_PATH.as_str()).await + let mut flock_cmd = Command::new(FLOCK_PATH.as_str()); + flock_cmd + .env_clear() + .envs(PROXY_ENVS.clone()) + .envs(envs) + .args([ + "-x", + &format!( + "{}/{}-{}.lock", + LOCK_CACHE_DIR, + if no_uv_install { "pip" } else { "py311" }, + fssafe_req + ), + "--command", + &command_args.join(" "), + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + start_child_process(flock_cmd, FLOCK_PATH.as_str()).await } else { let mut cmd = Command::new(command_args[0]); - cmd - .env_clear() + cmd.env_clear() .envs(PROXY_ENVS.clone()) .envs(envs) .args(&command_args[1..]) @@ -1321,7 +1341,6 @@ pub async fn handle_python_reqs( mut no_uv_install: bool, is_ansible: bool, ) -> error::Result> { - let counter_arc = Arc::new(tokio::sync::Mutex::new(0)); // Append logs with line like this: // [9/21] + requests==2.32.3 << (S3) | in 57ms @@ -1405,7 +1424,6 @@ pub async fn handle_python_reqs( .map(handle_ephemeral_token), ); - // Prepare NSJAIL if !*DISABLE_NSJAIL { let _ = write_file( @@ -1438,7 +1456,7 @@ pub async fn handle_python_reqs( let mut in_cache = vec![]; for req in requirements { // Ignore python version annotation backed into lockfile - if req.starts_with('#') { + if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() { continue; } // TODO: Remove @@ -1461,12 +1479,19 @@ pub async fn handle_python_reqs( } } if in_cache.len() > 0 { - append_logs(&job_id, w_id, format!("\nenv deps from local cache: {}\n", in_cache.join(", ")), db).await; + append_logs( + &job_id, + w_id, + format!("\nenv deps from local cache: {}\n", in_cache.join(", ")), + db, + ) + .await; } let (kill_tx, ..) = tokio::sync::broadcast::channel::<()>(1); - let kill_rxs: Vec> = - (0..req_with_penv.len()).map(|_| kill_tx.subscribe()).collect(); + let kill_rxs: Vec> = (0..req_with_penv.len()) + .map(|_| kill_tx.subscribe()) + .collect(); // ________ Read comments at the end of the function to get more context let (_done_tx, mut done_rx) = tokio::sync::mpsc::channel::<()>(1); @@ -1506,21 +1531,21 @@ pub async fn handle_python_reqs( if canceled { tracing::info!( - // If there is listener on other side, + // If there is listener on other side, workspace_id = %w_id_2, "cancelling installations", ); if let Err(ref e) = kill_tx.send(()){ tracing::error!( - // If there is listener on other side, + // If there is listener on other side, workspace_id = %w_id_2, "failed to send done: Probably receiving end closed too early or have not opened yet\n{}", // If there is no listener, it will be dropped safely e ); } - } + } } // Once done_tx is dropped, this will be fired _ = done_rx.recv() => break @@ -1555,9 +1580,15 @@ pub async fn handle_python_reqs( // Do we use Nsjail? if !*DISABLE_NSJAIL { - logs.push_str(&format!("\nStarting isolated installation... ({} tasks in parallel) \n", parallel_limit)); + logs.push_str(&format!( + "\nStarting isolated installation... ({} tasks in parallel) \n", + parallel_limit + )); } else { - logs.push_str(&format!("\nStarting installation... ({} tasks in parallel) \n", parallel_limit)); + logs.push_str(&format!( + "\nStarting installation... ({} tasks in parallel) \n", + parallel_limit + )); } append_logs(&job_id, w_id, logs, db).await; } @@ -1617,13 +1648,12 @@ pub async fn handle_python_reqs( tokio::select! { // Cancel was called on the job _ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")), - pull = pull_from_tar(os, venv_p.clone(), no_uv_install) => { if let Err(e) = pull { tracing::info!( workspace_id = %w_id, "No tarball was found on S3 or different problem occured {job_id}:\n{e}", - ); + ); } else { print_success( true, @@ -1653,12 +1683,12 @@ pub async fn handle_python_reqs( no_uv_install, ).await { Ok(r) => r, - Err(e) => { + Err(e) => { append_logs( &job_id, w_id, format!( - "\nError while spawning proccess:\n{e}", + "\nError while spawning proccess:\n{e}", ), db, ) @@ -1698,7 +1728,7 @@ pub async fn handle_python_reqs( &job_id, w_id, format!( - "\nError while installing {}:\n{buf}", + "\nError while installing {}:\n{buf}", &req ), db, @@ -1757,7 +1787,10 @@ pub async fn handle_python_reqs( let mut failed = false; for (handle, (_, venv_p)) in handles.into_iter().zip(req_with_penv.into_iter()) { - if let Err(e) = handle.await.unwrap_or(Err(anyhow!("Problem by joining handle"))) { + if let Err(e) = handle + .await + .unwrap_or(Err(anyhow!("Problem by joining handle"))) + { failed = true; tracing::warn!( workspace_id = %w_id, @@ -1778,19 +1811,11 @@ pub async fn handle_python_reqs( if has_work { let total_time = total_time.elapsed().as_millis(); - append_logs( - &job_id, - w_id, - format!( - "\nenv set in {}ms", - total_time - ), - db, - ).await; + append_logs(&job_id, w_id, format!("\nenv set in {}ms", total_time), db).await; } // Usually done_tx will drop after this return - // If there is listener on other side, + // If there is listener on other side, // it will be triggered // If there is no listener, it will be dropped safely return if failed { diff --git a/backend/windmill-worker/src/result_processor.rs b/backend/windmill-worker/src/result_processor.rs index 25bfade2e8..031cff0544 100644 --- a/backend/windmill-worker/src/result_processor.rs +++ b/backend/windmill-worker/src/result_processor.rs @@ -1,3 +1,6 @@ +#[cfg(feature = "otel")] +use opentelemetry::trace::FutureExt; + use serde::Serialize; use sqlx::{types::Json, Pool, Postgres}; use std::{ @@ -7,6 +10,9 @@ use std::{ Arc, }, }; +use tracing::{field, Instrument}; +#[cfg(not(feature = "otel"))] +use windmill_common::otel_ee::FutureExt; use uuid::Uuid; @@ -85,7 +91,49 @@ pub fn start_background_processor( JobKind::Dependencies | JobKind::FlowDependencies ); - handle_receive_completed_job( + let success = jc.success; + + let span = tracing::span!( + tracing::Level::INFO, + "job_postprocessing", + job_id = %jc.job.id, root_job = field::Empty, workspace_id = %jc.job.workspace_id, worker = %worker_name,tag = %jc.job.tag, + // hostname = %hostname, + language = field::Empty, + script_path = field::Empty, + flow_step_id = field::Empty, + parent_job = field::Empty, + otel.name = field::Empty + ); + let rj = if let Some(root_job) = jc.job.root_job { + root_job + } else { + jc.job.id + }; + windmill_common::otel_ee::set_span_parent(&span, &rj); + + if let Some(lg) = jc.job.language.as_ref() { + span.record("language", lg.as_str()); + } + if let Some(step_id) = jc.job.flow_step_id.as_ref() { + span.record( + "otel.name", + format!("job_postprocessing {}", step_id).as_str(), + ); + span.record("flow_step_id", step_id.as_str()); + } else { + span.record("otel.name", "job postprocessing"); + } + if let Some(parent_job) = jc.job.parent_job.as_ref() { + span.record("parent_job", parent_job.to_string().as_str()); + } + if let Some(script_path) = jc.job.script_path.as_ref() { + span.record("script_path", script_path.as_str()); + } + if let Some(root_job) = jc.job.root_job.as_ref() { + span.record("root_job", root_job.to_string().as_str()); + } + + let root_job = handle_receive_completed_job( jc, &base_internal_url, &db, @@ -96,8 +144,13 @@ pub fn start_background_processor( #[cfg(feature = "benchmark")] &mut bench, ) + .instrument(span) .await; + if let Some(root_job) = root_job { + windmill_common::otel_ee::add_root_flow_job_to_otlp(&root_job, success); + } + if is_init_script_and_failure { tracing::error!("init script errored, exiting"); killpill_tx.send(()).unwrap_or_default(); @@ -199,7 +252,11 @@ async fn send_job_completed( token, duration, }; - job_completed_tx.send(jc).await.expect("send job completed") + job_completed_tx + .send(jc) + .with_context(windmill_common::otel_ee::otel_ctx()) + .await + .expect("send job completed") } pub async fn process_result( @@ -271,6 +328,7 @@ pub async fn process_result( token, duration, ) + .with_context(windmill_common::otel_ee::otel_ctx()) .await; Ok(true) } @@ -315,6 +373,7 @@ pub async fn process_result( token, duration, ) + .with_context(windmill_common::otel_ee::otel_ctx()) .await; Ok(false) } @@ -330,7 +389,7 @@ pub async fn handle_receive_completed_job( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) { +) -> Option> { let token = jc.token.clone(); let workspace = jc.job.workspace_id.clone(); let client = AuthedClient { @@ -342,7 +401,7 @@ pub async fn handle_receive_completed_job( let job = jc.job.clone(); let mem_peak = jc.mem_peak.clone(); let canceled_by = jc.canceled_by.clone(); - if let Err(err) = process_completed_job( + match process_completed_job( jc, &client, db, @@ -355,26 +414,29 @@ pub async fn handle_receive_completed_job( ) .await { - handle_job_error( - db, - &client, - job.as_ref(), - mem_peak, - canceled_by, - err, - false, - same_worker_tx.clone(), - &worker_dir, - worker_name, - job_completed_tx, - #[cfg(feature = "benchmark")] - bench, - ) - .await; + Err(err) => { + handle_job_error( + db, + &client, + job.as_ref(), + mem_peak, + canceled_by, + err, + false, + same_worker_tx.clone(), + &worker_dir, + worker_name, + job_completed_tx, + #[cfg(feature = "benchmark")] + bench, + ) + .await; + None + } + Ok(r) => r, } } -#[tracing::instrument(name = "completed_job", level = "info", skip_all, fields(job_id = %job.id))] pub async fn process_completed_job( JobCompleted { job, result, mem_peak, success, cached_res_path, canceled_by, duration, .. }: JobCompleted, client: &AuthedClient, @@ -384,11 +446,11 @@ pub async fn process_completed_job( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) -> windmill_common::error::Result<()> { +) -> error::Result>> { if success { // println!("bef completed job{:?}", SystemTime::now()); if let Some(cached_path) = cached_res_path { - save_in_cache(db, client, &job, cached_path.to_string(), &result).await; + save_in_cache(db, client, &job, cached_path, result.clone()).await; } let is_flow_step = job.is_flow_step; @@ -414,8 +476,8 @@ pub async fn process_completed_job( if is_flow_step { if let Some(parent_job) = parent_job { - tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)"); - update_flow_status_after_job_completion( + // tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)"); + let r = update_flow_status_after_job_completion( db, client, parent_job, @@ -434,9 +496,10 @@ pub async fn process_completed_job( ) .warn_after_seconds(10) .await?; + add_time!(bench, "updated flow status END"); + return Ok(r); } } - add_time!(bench, "updated flow status END"); } else { let result = add_completed_job_error( db, @@ -454,7 +517,7 @@ pub async fn process_completed_job( if job.is_flow_step { if let Some(parent_job) = job.parent_job { tracing::error!(parent_flow = %parent_job, subflow = %job.id, "process completed job error, updating flow status"); - update_flow_status_after_job_completion( + let r = update_flow_status_after_job_completion( db, client, parent_job, @@ -473,10 +536,11 @@ pub async fn process_completed_job( ) .warn_after_seconds(10) .await?; + return Ok(r); } } } - Ok(()) + return Ok(None); } #[tracing::instrument(name = "job_error", level = "info", skip_all, fields(job_id = %job.id))] diff --git a/backend/windmill-worker/src/snowflake_executor.rs b/backend/windmill-worker/src/snowflake_executor.rs index 9f8d3e12fd..4e817a3927 100644 --- a/backend/windmill-worker/src/snowflake_executor.rs +++ b/backend/windmill-worker/src/snowflake_executor.rs @@ -419,6 +419,7 @@ pub async fn do_snowflake( worker_name, &job.workspace_id, &mut Some(occupancy_metrics), + Box::pin(futures::stream::once(async { 0 })), ) .await?; *mem_peak = (r.get().len() / 1000) as i32; diff --git a/backend/windmill-worker/src/windmill-client.js b/backend/windmill-worker/src/windmill-client.js index 64281fede9..3ae15e7453 100644 --- a/backend/windmill-worker/src/windmill-client.js +++ b/backend/windmill-worker/src/windmill-client.js @@ -558,6 +558,9 @@ var $QueuedJob = { "identity", "deploymentcallback", "singlescriptflow", + "flowscript", + "flownode", + "appscript", ], }, schedule_path: { @@ -697,6 +700,9 @@ var $CompletedJob = { "identity", "deploymentcallback", "singlescriptflow", + "flowscript", + "flownode", + "appscript", ], }, schedule_path: { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 36430b5495..08464551a2 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -6,7 +6,11 @@ * LICENSE-AGPL for a copy of the license. */ +// #[cfg(feature = "otel")] +// use opentelemetry::{global, KeyValue}; + use windmill_common::{ + apps::AppScriptId, auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET}, scripts::PREVIEW_IS_TAR_CODEBASE_HASH, utils::WarnAfterExt, @@ -24,7 +28,7 @@ use const_format::concatcp; #[cfg(feature = "prometheus")] use prometheus::IntCounter; -use tracing::Instrument; +use tracing::{field, Instrument}; #[cfg(feature = "prometheus")] use windmill_common::METRICS_DEBUG_ENABLED; #[cfg(feature = "prometheus")] @@ -34,9 +38,8 @@ use reqwest::Response; use serde::{de::DeserializeOwned, Deserialize, Serialize}; use sqlx::{types::Json, Pool, Postgres}; use std::{ - collections::{hash_map::DefaultHasher, HashMap}, + collections::HashMap, fs::DirBuilder, - hash::Hash, sync::{ atomic::{AtomicBool, AtomicU16, Ordering}, Arc, @@ -91,8 +94,8 @@ use crate::{ bash_executor::{handle_bash_job, handle_powershell_job}, bun_executor::handle_bun_job, common::{ - build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args, - update_worker_ping_for_failed_init_script, OccupancyMetrics, + build_args_map, cached_result_path, get_cached_resource_value_if_valid, + get_reserved_variables, update_worker_ping_for_failed_init_script, OccupancyMetrics, }, csharp_executor::handle_csharp_job, deno_executor::handle_deno_job, @@ -102,18 +105,20 @@ use crate::{ handle_job_error, job_logger::NO_LOGS_AT_ALL, js_eval::{eval_fetch_timeout, transpile_ts}, - mysql_executor::do_mysql, pg_executor::do_postgresql, php_executor::handle_php_job, python_executor::handle_python_job, result_processor::{process_result, start_background_processor}, rust_executor::handle_rust_job, - worker_flow::{handle_flow, update_flow_status_in_progress, Step}, + worker_flow::{handle_flow, update_flow_status_in_progress}, worker_lockfiles::{ handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job, }, }; +#[cfg(feature = "mysql")] +use crate::mysql_executor::do_mysql; + use backon::ConstantBuilder; use backon::{BackoffBuilder, Retryable}; @@ -722,7 +727,10 @@ fn add_outstanding_wait_time( }.in_current_span()); } -#[tracing::instrument(name = "worker", level = "info", skip_all, fields(worker = %worker_name, hostname = %hostname))] +// struct WorkerMtrics { +// job_ +// } + pub async fn run_worker( db: &Pool, hostname: &str, @@ -738,6 +746,7 @@ pub async fn run_worker( #[cfg(not(feature = "enterprise"))] if !*DISABLE_NSJAIL { tracing::warn!( + worker = %worker_name, hostname = %hostname, "NSJAIL to sandbox process in untrusted environments is an enterprise feature but allowed to be used for testing purposes" ); } @@ -745,10 +754,10 @@ pub async fn run_worker( let start_time = Instant::now(); let worker_dir = format!("{TMP_DIR}/{worker_name}"); - tracing::debug!(worker_dir = %worker_dir, "Creating worker dir"); + tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir"); if let Some(ref netrc) = *NETRC { - tracing::info!("Writing netrc at {}/.netrc", HOME_ENV.as_str()); + tracing::info!(worker = %worker_name, hostname = %hostname, "Writing netrc at {}/.netrc", HOME_ENV.as_str()); write_file(&HOME_ENV, ".netrc", netrc).expect("could not write netrc"); } @@ -963,6 +972,14 @@ pub async fn run_worker( None }; + // let worker_resource = &[ + // KeyValue::new("hostname", hostname.to_string()), + // KeyValue::new("worker", worker_name.to_string()), + // ]; + // // Create a meter from the above MeterProvider. + // let meter = global::meter("windmill"); + // let counter = meter.u64_counter("jobs.execution").build(); + let mut occupancy_metrics = OccupancyMetrics::new(start_time); let mut jobs_executed = 0; @@ -1018,6 +1035,7 @@ pub async fn run_worker( IS_READY.store(true, Ordering::Relaxed); tracing::info!( + worker = %worker_name, hostname = %hostname, "listening for jobs, WORKER_GROUP: {}, config: {:?}", *WORKER_GROUP, WORKER_CONFIG.read().await @@ -1053,7 +1071,7 @@ pub async fn run_worker( if i_worker == 1 { if let Err(e) = queue_init_bash_maybe(db, same_worker_tx.clone(), &worker_name).await { killpill_tx.send(()).unwrap_or_default(); - tracing::error!("Error queuing init bash script for worker {worker_name}: {e:#}"); + tracing::error!(worker = %worker_name, hostname = %hostname, "Error queuing init bash script for worker {worker_name}: {e:#}"); return; } } @@ -1082,11 +1100,12 @@ pub async fn run_worker( let mut last_suspend_first = Instant::now(); let mut killed_but_draining_same_worker_jobs = false; + let mut killpill_rx2 = killpill_rx.resubscribe(); loop { #[cfg(feature = "enterprise")] { if let Ok(_) = killpill_rx.try_recv() { - tracing::info!("killpill received on worker waiting for valid key"); + tracing::info!(worker = %worker_name, hostname = %hostname, "killpill received on worker waiting for valid key"); job_completed_tx .0 .send(SendResult::Kill) @@ -1098,6 +1117,7 @@ pub async fn run_worker( if !valid_key { tracing::error!( + worker = %worker_name, hostname = %hostname, "Invalid license key, workers require a valid license key, sleeping for 30s waiting for valid key to be set" ); tokio::time::sleep(Duration::from_secs(10)).await; @@ -1111,7 +1131,7 @@ pub async fn run_worker( #[cfg(feature = "prometheus")] if let Some(wk) = worker_busy.as_ref() { wk.set(0); - tracing::debug!("set worker busy to 0"); + tracing::debug!(worker = %worker_name, hostname = %hostname, "set worker busy to 0"); } occupancy_metrics.running_job_started_at = None; @@ -1123,7 +1143,7 @@ pub async fn run_worker( .try_into() .unwrap(), ); - tracing::debug!("set uptime metric"); + tracing::debug!(worker = %worker_name, hostname = %hostname, "set uptime metric"); } if last_ping.elapsed().as_secs() > NUM_SECS_PING { @@ -1167,15 +1187,19 @@ pub async fn run_worker( ) .notify(|err, dur| { tracing::error!( + worker = %worker_name, hostname = %hostname, "retrying updating worker ping in {dur:#?}, err: {err:#?}" ); }) .sleep(tokio::time::sleep) .await { - tracing::error!("failed to update worker ping, exiting: {}", e); + tracing::error!( + worker = %worker_name, hostname = %hostname, + "failed to update worker ping, exiting: {}", e); killpill_tx.send(()).unwrap_or_default(); } tracing::info!( + worker = %worker_name, hostname = %hostname, "ping update, memory: container={}MB, windmill={}MB", memory_usage.unwrap_or_default() / (1024 * 1024), wm_memory_usage.unwrap_or_default() / (1024 * 1024) @@ -1187,16 +1211,18 @@ pub async fn run_worker( if (jobs_executed as u32 + vacuum_shift) % VACUUM_PERIOD == 0 { let db2 = db.clone(); let current_span = tracing::Span::current(); + let worker_name = worker_name.clone(); + let hostname = hostname.to_string(); tokio::task::spawn( (async move { - tracing::info!("vacuuming queue and completed_job"); + tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue"); if let Err(e) = sqlx::query!("VACUUM (skip_locked) queue") .execute(&db2) .await { - tracing::error!("failed to vacuum queue: {}", e); + tracing::error!(worker = %worker_name, hostname = %hostname, "failed to vacuum queue: {}", e); } - tracing::info!("vacuumed queue and completed_job"); + tracing::info!(worker = %worker_name, hostname = %hostname, "vacuumed queue"); }) .instrument(current_span), ); @@ -1232,6 +1258,7 @@ pub async fn run_worker( if let Ok(same_worker_job) = same_worker_rx.try_recv() { same_worker_queue_size.fetch_sub(1, Ordering::SeqCst); tracing::debug!( + worker = %worker_name, hostname = %hostname, "received {} from same worker channel", same_worker_job.job_id ); @@ -1244,6 +1271,7 @@ pub async fn run_worker( .map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string())); if r.is_err() && !same_worker_job.recoverable { tracing::error!( + worker = %worker_name, hostname = %hostname, "failed to fetch same_worker job on a non recoverable job, exiting" ); job_completed_tx @@ -1257,7 +1285,7 @@ pub async fn run_worker( } } else if let Ok(_) = killpill_rx.try_recv() { if !killed_but_draining_same_worker_jobs { - tracing::info!("received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker); + tracing::info!(worker = %worker_name, hostname = %hostname, "received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker); killed_but_draining_same_worker_jobs = true; job_completed_tx .0 @@ -1268,10 +1296,10 @@ pub async fn run_worker( continue; } else if killed_but_draining_same_worker_jobs { if job_completed_processor_is_done.load(Ordering::SeqCst) { - tracing::info!("all running jobs have completed and all completed jobs have been fully processed, exiting"); + tracing::info!(worker = %worker_name, hostname = %hostname, "all running jobs have completed and all completed jobs have been fully processed, exiting"); break; } else { - tracing::info!("there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting"); + tracing::info!(worker = %worker_name, hostname = %hostname, "there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting"); tokio::time::sleep(Duration::from_millis(200)).await; continue; } @@ -1296,7 +1324,7 @@ pub async fn run_worker( if !agent_mode && duration_pull_s > 0.5 { let empty = job.as_ref().is_ok_and(|x| x.0.is_none()); - tracing::warn!("pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}"); + tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}"); #[cfg(feature = "prometheus")] if empty { if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() { @@ -1307,7 +1335,7 @@ pub async fn run_worker( } } else if !agent_mode && duration_pull_s > 0.1 { let empty = job.as_ref().is_ok_and(|x| x.0.is_none()); - tracing::warn!("pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}"); + tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}"); #[cfg(feature = "prometheus")] if empty { if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() { @@ -1361,7 +1389,7 @@ pub async fn run_worker( last_executed_job = None; jobs_executed += 1; - tracing::debug!("started handling of job {}", job.id); + tracing::debug!(worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id); if matches!(job.job_kind, JobKind::Script | JobKind::Preview) { if !dedicated_workers.is_empty() { @@ -1426,6 +1454,11 @@ pub async fn run_worker( ) .await; + // counter.add( + // 1, + // worker_resource + // ); + #[cfg(feature = "prometheus")] let _timer = register_metric( &WORKER_EXECUTION_DURATION, @@ -1513,6 +1546,39 @@ pub async fn run_worker( let PulledJob { job, raw_code, raw_lock, raw_flow } = job; let arc_job = Arc::new(job); add_time!(bench, "handle_queued_job START"); + + let span = tracing::span!(tracing::Level::INFO, "job", + job_id = %arc_job.id, root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, hostname = %hostname, tag = %arc_job.tag, + language = field::Empty, + script_path = field::Empty, flow_step_id = field::Empty, parent_job = field::Empty, + otel.name = field::Empty); + let rj = if let Some(root_job) = arc_job.root_job { + root_job + } else { + arc_job.id + }; + if let Some(lg) = arc_job.language.as_ref() { + span.record("language", lg.as_str()); + } + if let Some(step_id) = arc_job.flow_step_id.as_ref() { + span.record("otel.name", format!("job {}", step_id).as_str()); + span.record("flow_step_id", step_id.as_str()); + } else { + span.record("otel.name", "job"); + } + if let Some(parent_job) = arc_job.parent_job.as_ref() { + span.record("parent_job", parent_job.to_string().as_str()); + } + if let Some(script_path) = arc_job.script_path.as_ref() { + span.record("script_path", script_path.as_str()); + } + if let Some(root_job) = arc_job.root_job.as_ref() { + span.record("root_job", root_job.to_string().as_str()); + } + + windmill_common::otel_ee::set_span_parent(&span, &rj); + // span.context().span().add_event_with_timestamp("job created".to_string(), arc_job.created_at.into(), vec![]); + match handle_queued_job( arc_job.clone(), raw_code, @@ -1528,9 +1594,11 @@ pub async fn run_worker( base_internal_url, job_completed_tx.clone(), &mut occupancy_metrics, + &mut killpill_rx2, #[cfg(feature = "benchmark")] &mut bench, ) + .instrument(span) .await { Err(err) => { @@ -1609,7 +1677,7 @@ pub async fn run_worker( if let Some(secs) = *EXIT_AFTER_NO_JOB_FOR_SECS { if let Some(lj) = last_executed_job { if lj.elapsed().as_secs() > secs { - tracing::info!("no job for {} seconds, exiting", secs); + tracing::info!(worker = %worker_name, hostname = %hostname, "no job for {} seconds, exiting", secs); break; } } else { @@ -1640,12 +1708,12 @@ pub async fn run_worker( }); } Err(err) => { - tracing::error!("Failed to pull jobs: {}", err); + tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err); } }; } - tracing::info!("worker {} exiting", worker_name); + tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exiting", worker_name); #[cfg(feature = "benchmark")] { @@ -1660,20 +1728,20 @@ pub async fn run_worker( if has_dedicated_workers { for handle in dedicated_handles { if let Err(e) = handle.await { - tracing::error!("error in dedicated worker waiting for it to end: {:?}", e) + tracing::error!(worker = %worker_name, hostname = %hostname, "error in dedicated worker waiting for it to end: {:?}", e) } } - tracing::info!("all dedicated workers have exited"); + tracing::info!(worker = %worker_name, hostname = %hostname, "all dedicated workers have exited"); } drop(job_completed_tx); - tracing::info!("waiting for job_completed_processor to finish processing remaining jobs"); + tracing::info!(worker = %worker_name, hostname = %hostname, "waiting for job_completed_processor to finish processing remaining jobs"); if let Err(e) = send_result.await { tracing::error!("error in awaiting send_result process: {e:?}") } - tracing::info!("worker {} exited", worker_name); - tracing::info!("number of jobs executed: {}", jobs_executed); + tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name); + tracing::info!(worker = %worker_name, hostname = %hostname, "number of jobs executed: {}", jobs_executed); } async fn queue_init_bash_maybe<'c>( @@ -1800,7 +1868,6 @@ pub struct PreviousResult<'a> { pub previous_result: Option<&'a RawValue>, } -#[tracing::instrument(name = "job", level = "info", skip_all, fields(job_id = %job.id))] async fn handle_queued_job( job: Arc, raw_code: Option, @@ -1816,8 +1883,11 @@ async fn handle_queued_job( base_internal_url: &str, job_completed_tx: JobCompletedSender, occupancy_metrics: &mut OccupancyMetrics, + killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, ) -> windmill_common::error::Result { + // Extract the active span from the context + if job.canceled { return Err(Error::JsonErr(canceled_job_to_result(&job))); } @@ -1857,8 +1927,8 @@ async fn handle_queued_job( } } - let step = if job.is_flow_step { - let r = update_flow_status_in_progress( + if job.is_flow_step { + let _ = update_flow_status_in_progress( db, &job.workspace_id, job.parent_job @@ -1867,24 +1937,19 @@ async fn handle_queued_job( ) .warn_after_seconds(5) .await?; - - Some(r) - } else { - if let Some(parent_job) = job.parent_job { - if let Err(e) = sqlx::query_scalar!( - "UPDATE queue SET flow_status = jsonb_set(jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], COALESCE(flow_status->$1, '{}'::jsonb)), array[$1, 'started_at'], to_jsonb(now()::text)) WHERE id = $2 AND workspace_id = $3", - &job.id.to_string(), - parent_job, - &job.workspace_id - ) - .execute(db) - .warn_after_seconds(5) - .await { - tracing::error!("Could not update parent job started_at flow_status: {}", e); - } + } else if let Some(parent_job) = job.parent_job { + if let Err(e) = sqlx::query_scalar!( + "UPDATE queue SET flow_status = jsonb_set(jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], COALESCE(flow_status->$1, '{}'::jsonb)), array[$1, 'started_at'], to_jsonb(now()::text)) WHERE id = $2 AND workspace_id = $3", + &job.id.to_string(), + parent_job, + &job.workspace_id + ) + .execute(db) + .warn_after_seconds(5) + .await { + tracing::error!("Could not update parent job started_at flow_status: {}", e); } - None - }; + } let started = Instant::now(); let (raw_code, raw_lock, raw_flow) = match (raw_code, raw_lock, raw_flow) { @@ -1903,68 +1968,25 @@ async fn handle_queued_job( }; let cached_res_path = if job.cache_ttl.is_some() { - let version_hash = if let Some(h) = job.script_hash { - if matches!(job.job_kind, JobKind::FlowScript) { - format!("flowscript_{}", h.to_string()) - } else { - format!("script_{}", h.to_string()) - } - } else if let Some(rc) = raw_code.as_ref() { - use std::hash::Hasher; - let mut s = DefaultHasher::new(); - rc.hash(&mut s); - format!("inline_{}", hex::encode(s.finish().to_be_bytes())) - } else if let Some(sqlx::types::Json(rc)) = raw_flow.as_ref() { - use std::hash::Hasher; - let mut s = DefaultHasher::new(); - rc.get().hash(&mut s); - format!("flow_{}", hex::encode(s.finish().to_be_bytes())) - } else { - "none".to_string() - }; - let args_hash = hash_args( - db, - &client.get_authed().await, - &job.workspace_id, - &job.id, - &job.args, - ) - .await; - if job.is_flow_step && !matches!(job.job_kind, JobKind::Flow) { - let flow_path = sqlx::query_scalar!( - "SELECT script_path FROM queue WHERE id = $1", - &job.parent_job.unwrap() + Some( + cached_result_path( + db, + &client.get_authed().await, + &job, + raw_code.as_ref(), + raw_lock.as_ref(), + raw_flow.as_ref(), ) - .fetch_one(db) - .warn_after_seconds(5) - .await - .map_err(|e| { - Error::InternalErr(format!( - "Fetching script path from queue for caching purposes: {e:#}" - )) - })? - .ok_or_else(|| Error::InternalErr(format!("Expected script_path")))?; - let step = match step.unwrap() { - Step::Step(i) => i.to_string(), - Step::PreprocessorStep => "preprocessor".to_string(), - Step::FailureStep => "failure".to_string(), - }; - Some(format!( - "{flow_path}/cache/{version_hash}/{step}/{args_hash}" - )) - } else if let Some(script_path) = &job.script_path { - Some(format!("{script_path}/cache/{version_hash}/{args_hash}")) - } else { - None - } + .await, + ) } else { None }; - if let Some(cached_res_path) = cached_res_path.clone() { + if let Some(cached_res_path) = cached_res_path.as_ref() { let authed_client = client.get_authed().await; - let cached_resource_value_maybe = get_cached_resource_value_if_valid( + let cached_result_maybe = get_cached_resource_value_if_valid( db, &authed_client, &job.id, @@ -1973,7 +1995,7 @@ async fn handle_queued_job( ) .warn_after_seconds(5) .await; - if let Some(cached_resource_value) = cached_resource_value_maybe { + if let Some(result) = cached_result_maybe { { let logs = "Job skipped because args & path found in cache and not expired".to_string(); @@ -1981,8 +2003,8 @@ async fn handle_queued_job( } job_completed_tx .send(JobCompleted { - job: job, - result: Arc::new(cached_resource_value), + job, + result, mem_peak: 0, canceled_by: None, success: true, @@ -2123,6 +2145,7 @@ async fn handle_queued_job( &mut column_order, &mut new_args, occupancy_metrics, + killpill_rx, ) .await; occupancy_metrics.total_duration_of_running_jobs += @@ -2266,6 +2289,7 @@ async fn handle_code_execution_job( column_order: &mut Option>, new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, + killpill_rx: &mut tokio::sync::broadcast::Receiver<()>, ) -> error::Result> { let ContentReqLangEnvs { content: inner_content, @@ -2314,6 +2338,20 @@ async fn handle_code_execution_job( codebase: None, } } + JobKind::AppScript => { + let (lockfile, content) = cache::app::fetch_script( + db, + AppScriptId(job.script_hash.unwrap_or(ScriptHash(0)).0), + ) + .await?; + ContentReqLangEnvs { + content, + lockfile, + language: job.language.to_owned(), + envs: None, + codebase: None, + } + } JobKind::DeploymentCallback => { get_script_content_by_path(job.script_path.clone(), &job.workspace_id, db).await? } @@ -2336,6 +2374,10 @@ async fn handle_code_execution_job( ) .await; } else if language == Some(ScriptLang::Mysql) { + #[cfg(not(feature = "mysql"))] + return Err(Error::InternalErr("MySQL requires the mysql feature to be enabled".to_string())); + + #[cfg(feature = "mysql")] return do_mysql( job, &client, @@ -2595,6 +2637,7 @@ mount {{ worker_name, envs, occupancy_metrics, + killpill_rx, ) .await } diff --git a/backend/windmill-worker/src/worker_flow.rs b/backend/windmill-worker/src/worker_flow.rs index 050f0ebcf4..17c1e58859 100644 --- a/backend/windmill-worker/src/worker_flow.rs +++ b/backend/windmill-worker/src/worker_flow.rs @@ -6,13 +6,12 @@ * LICENSE-AGPL for a copy of the license. */ -use std::collections::hash_map::DefaultHasher; use std::collections::HashMap; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; -use crate::common::{hash_args, save_in_cache}; +use crate::common::{cached_result_path, save_in_cache}; use crate::js_eval::{eval_timeout, IdContext}; use crate::{ AuthedClient, PreviousResult, SameWorkerPayload, SameWorkerSender, SendResult, JOB_TOKEN, @@ -77,45 +76,34 @@ pub async fn update_flow_status_after_job_completion( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) -> error::Result<()> { +) -> error::Result>> { // this is manual tailrecursion because async_recursion blows up the stack - // todo!(); potentially_crash_for_testing(); - let mut rec = update_flow_status_after_job_completion_internal( - db, - client, + let mut rec = RecUpdateFlowStatusAfterJobCompletion { flow, - job_id_for_status, - w_id, + job_id_for_status: job_id_for_status.clone(), success, result, - unrecoverable, - same_worker_tx.clone(), - worker_dir, stop_early_override, - false, - worker_name, - job_completed_tx.clone(), - #[cfg(feature = "benchmark")] - bench, - ) - .await?; - while let Some(nrec) = rec { + skip_error_handler: false, + }; + let mut unrecoverable = unrecoverable; + loop { potentially_crash_for_testing(); - rec = match update_flow_status_after_job_completion_internal( + let nrec = match update_flow_status_after_job_completion_internal( db, client, - nrec.flow, - &nrec.job_id_for_status, + rec.flow, + &rec.job_id_for_status, w_id, - nrec.success, - nrec.result, - false, + rec.success, + rec.result, + unrecoverable, same_worker_tx.clone(), worker_dir, - nrec.stop_early_override, - nrec.skip_error_handler, + rec.stop_early_override, + rec.skip_error_handler, worker_name, job_completed_tx.clone(), #[cfg(feature = "benchmark")] @@ -125,12 +113,12 @@ pub async fn update_flow_status_after_job_completion( { Ok(j) => j, Err(e) => { - tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", nrec.flow,&nrec.job_id_for_status); + tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", rec.flow, &rec.job_id_for_status); update_flow_status_after_job_completion_internal( db, client, - nrec.flow, - &nrec.job_id_for_status, + rec.flow, + &rec.job_id_for_status, w_id, false, Arc::new(to_raw_value(&Json(&WrappedError { @@ -139,8 +127,8 @@ pub async fn update_flow_status_after_job_completion( true, same_worker_tx.clone(), worker_dir, - nrec.stop_early_override, - nrec.skip_error_handler, + rec.stop_early_override, + rec.skip_error_handler, worker_name, job_completed_tx.clone(), #[cfg(feature = "benchmark")] @@ -148,9 +136,33 @@ pub async fn update_flow_status_after_job_completion( ) .await? } + }; + unrecoverable = false; + match nrec { + UpdateFlowStatusAfterJobCompletion::Done(job) => { + add_time!(bench, "update flow status internal END"); + return Ok(Some(job)); + } + UpdateFlowStatusAfterJobCompletion::Rec(nrec) => { + rec = nrec; + } + UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch => { + add_time!(bench, "update flow status internal END"); + return Ok(None); + } + UpdateFlowStatusAfterJobCompletion::NotDone => { + add_time!(bench, "update flow status internal END"); + return Ok(None); + } } } - Ok(()) +} + +pub enum UpdateFlowStatusAfterJobCompletion { + Rec(RecUpdateFlowStatusAfterJobCompletion), + Done(Arc), + NotDone, + NonLastParallelBranch, } pub struct RecUpdateFlowStatusAfterJobCompletion { flow: uuid::Uuid, @@ -188,7 +200,7 @@ pub async fn update_flow_status_after_job_completion_internal( worker_name: &str, job_completed_tx: Sender, #[cfg(feature = "benchmark")] bench: &mut BenchmarkIter, -) -> error::Result> { +) -> error::Result { add_time!(bench, "update flow status internal START"); let ( should_continue_flow, @@ -272,93 +284,96 @@ pub async fn update_flow_status_after_job_completion_internal( let is_failure_step = old_status.step >= old_status.modules.len() as i32 && old_status.modules.len() > 0; - let (mut stop_early, mut skip_if_stop_early, continue_on_error) = if let Some(se) = - stop_early_override - { - //do not stop early if module is a flow step - let step = match module_step { - Step::PreprocessorStep => None, - Step::FailureStep => None, - Step::Step(i) => Some(i), - }; + let (mut stop_early, mut skip_if_stop_early, continue_on_error) = + if let Some(se) = stop_early_override { + //do not stop early if module is a flow step + let step = match module_step { + Step::PreprocessorStep => None, + Step::FailureStep => None, + Step::Step(i) => Some(i), + }; - let is_flow = if let Some(_) = step { - #[derive(Deserialize)] - struct GetType<'j> { r#type: &'j str } + let is_flow = if let Some(_) = step { + #[derive(Deserialize)] + struct GetType<'j> { + r#type: &'j str, + } - current_module - .as_ref() - .map(|module| serde_json::from_str::(module.value.get()).map(|v| v.r#type == "flow")) - .unwrap_or(Ok(false)) - .unwrap_or(false) - } else { - false - }; - - if is_flow { - (false, false, false) - } else { - (true, se, false) - } - } else if is_failure_step || matches!(module_step, Step::PreprocessorStep) { - (false, false, false) - } else if let Some(current_module) = current_module.as_ref() { - let stop_early = success - && !is_branch_all - && if let Some(ref expr) = current_module - .stop_after_if - .as_ref() - .map(|x| x.expr.clone()) - { - let all_iters = match &module_status { - FlowStatusModule::InProgress { flow_jobs: Some(flow_jobs), .. } - if expr.contains("all_iters") => - { - Some(Arc::new( - retrieve_flow_jobs_results(db, w_id, flow_jobs).await?, - )) - } - _ => None, - }; - let args = sqlx::query_as::<_, RowArgs>( - "SELECT - args - FROM queue - WHERE id = $2", - ) - .bind(old_status.step) - .bind(flow) - .fetch_one(db) - .await - .map_err(|e| { - Error::InternalErr(format!("retrieval of args from state: {e:#}")) - })?; - compute_bool_from_expr( - &expr, - Marc::new(args.args.unwrap_or_default().0), - result.clone(), - all_iters, - None, - Some(client), - None, - None, - ) - .await? + current_module + .as_ref() + .map(|module| { + serde_json::from_str::(module.value.get()) + .map(|v| v.r#type == "flow") + }) + .unwrap_or(Ok(false)) + .unwrap_or(false) } else { false }; - ( - stop_early, - current_module - .stop_after_if - .as_ref() - .map(|x| x.skip_if_stopped) - .unwrap_or(false), - current_module.continue_on_error.unwrap_or(false), - ) - } else { - (false, false, false) - }; + + if is_flow { + (false, false, false) + } else { + (true, se, false) + } + } else if is_failure_step || matches!(module_step, Step::PreprocessorStep) { + (false, false, false) + } else if let Some(current_module) = current_module.as_ref() { + let stop_early = success + && !is_branch_all + && if let Some(ref expr) = current_module + .stop_after_if + .as_ref() + .map(|x| x.expr.clone()) + { + let all_iters = + match &module_status { + FlowStatusModule::InProgress { + flow_jobs: Some(flow_jobs), .. + } if expr.contains("all_iters") => Some(Arc::new( + retrieve_flow_jobs_results(db, w_id, flow_jobs).await?, + )), + _ => None, + }; + let args = sqlx::query_as::<_, RowArgs>( + "SELECT + args + FROM queue + WHERE id = $2", + ) + .bind(old_status.step) + .bind(flow) + .fetch_one(db) + .await + .map_err(|e| { + Error::InternalErr(format!("retrieval of args from state: {e:#}")) + })?; + compute_bool_from_expr( + &expr, + Marc::new(args.args.unwrap_or_default().0), + result.clone(), + all_iters, + None, + Some(client), + None, + None, + ) + .await? + } else { + false + }; + ( + stop_early, + current_module + .stop_after_if + .as_ref() + .map(|x| x.skip_if_stopped) + .unwrap_or(false), + current_module.continue_on_error.unwrap_or(false), + ) + } else { + (false, false, false) + }; let skip_branch_failure = match module_status { FlowStatusModule::InProgress { @@ -420,7 +435,7 @@ pub async fn update_flow_status_after_job_completion_internal( None }; - let nindex = if let Some(position) = position { + let nindex = if let Some(position) = position { sqlx::query_scalar!( "UPDATE queue SET flow_status = JSONB_SET( @@ -467,7 +482,7 @@ pub async fn update_flow_status_after_job_completion_internal( None }; - let nindex = if let Some(position) = position { + let nindex = if let Some(position) = position { sqlx::query_scalar!( "UPDATE queue SET flow_status = JSONB_SET( @@ -603,7 +618,7 @@ pub async fn update_flow_status_after_job_completion_internal( ); } add_time!(bench, "non final parallel flow finished"); - return Ok(None); + return Ok(UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch); } } FlowStatusModule::InProgress { @@ -677,16 +692,15 @@ pub async fn update_flow_status_after_job_completion_internal( } } if success || (flow_jobs.is_some() && (skip_loop_failures || skip_branch_failure)) { - let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some()) { + let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some()) + { sqlx::query_scalar!( "SELECT job_kind = 'identity' FROM completed_job WHERE id = $1", job_id_for_status ) .fetch_one(db) .await - .map_err(|e| { - Error::InternalErr(format!("error during skip check: {e:#}")) - })? + .map_err(|e| Error::InternalErr(format!("error during skip check: {e:#}")))? .unwrap_or(false) } else { false @@ -905,13 +919,15 @@ pub async fn update_flow_status_after_job_completion_internal( .context("remove flow status retry")?; } - let flow_job = sqlx::query_as::<_, PulledJob>("SELECT * FROM queue WHERE id = $1 AND workspace_id = $2") - .bind(flow) - .bind(w_id) - .fetch_optional(&mut *tx) - .await - .map_err(Into::::into)? - .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?; + let flow_job = sqlx::query_as::<_, PulledJob>( + "SELECT * FROM queue WHERE id = $1 AND workspace_id = $2", + ) + .bind(flow) + .bind(w_id) + .fetch_optional(&mut *tx) + .await + .map_err(Into::::into)? + .ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?; tx.commit().await?; let job_root = flow_job @@ -927,7 +943,8 @@ pub async fn update_flow_status_after_job_completion_internal( sqlx::query_scalar!( "SELECT raw_flow AS \"raw_flow!: Json>\" FROM job WHERE id = $1 AND workspace_id = $2 LIMIT 1", - &flow_job.id, w_id + &flow_job.id, + w_id ) .fetch_one(db) .await @@ -950,9 +967,16 @@ pub async fn update_flow_status_after_job_completion_internal( flow_value .as_ref() .and_then(|value| match module_step { - Step::PreprocessorStep => value.preprocessor_module.as_ref().and_then(|m| m.retry.as_ref()), - Step::Step(i) => value.modules.get(i).as_ref().and_then(|m| m.retry.as_ref()), - Step::FailureStep => value.failure_module.as_ref().and_then(|m| m.retry.as_ref()), + Step::PreprocessorStep => value + .preprocessor_module + .as_ref() + .and_then(|m| m.retry.as_ref()), + Step::Step(i) => { + value.modules.get(i).as_ref().and_then(|m| m.retry.as_ref()) + } + Step::FailureStep => { + value.failure_module.as_ref().and_then(|m| m.retry.as_ref()) + } }) .unwrap_or(&Retry::default()), &old_status.retry, @@ -964,7 +988,10 @@ pub async fn update_flow_status_after_job_completion_internal( false if !is_failure_step && !skip_error_handler - && flow_value.as_ref().map(|v| v.failure_module.is_some()).unwrap_or(false) => + && flow_value + .as_ref() + .map(|v| v.failure_module.is_some()) + .unwrap_or(false) => { true } @@ -1032,29 +1059,17 @@ pub async fn update_flow_status_after_job_completion_internal( reason: flow_job.canceled_reason.clone(), }), canceled_job_to_result(&flow_job), - worker_name, + worker_name, true, None, ) .await?; } else { if flow_job.cache_ttl.is_some() && success { - let cached_res_path = { - let args_hash = - hash_args(db, client, w_id, job_id_for_status, &flow_job.args).await; - let flow_path = flow_job.script_path(); - let version_hash = if let Some(sqlx::types::Json(s)) = raw_flow.as_ref() { - use std::hash::{Hash, Hasher}; - let mut h = DefaultHasher::new(); - s.get().hash(&mut h); - format!("flow_{}", hex::encode(h.finish().to_be_bytes())) - } else { - "flow_unknown".to_string() - }; - format!("{flow_path}/cache/{version_hash}/{args_hash}") - }; + let cached_res_path = + cached_result_path(db, client, &flow_job, None, None, raw_flow.as_ref()).await; - save_in_cache(db, client, &flow_job, cached_res_path, &nresult).await; + save_in_cache(db, client, &flow_job, cached_res_path, nresult.clone()).await; } fn result_has_recover_true(nresult: Arc>) -> bool { let recover = serde_json::from_str::(nresult.get()); @@ -1108,7 +1123,7 @@ pub async fn update_flow_status_after_job_completion_internal( Some(nresult.clone()), same_worker_tx.clone(), worker_dir, - job_completed_tx, + job_completed_tx, ) .warn_after_seconds(10) .await @@ -1122,17 +1137,8 @@ pub async fn update_flow_status_after_job_completion_internal( db, ) .await; - let _ = add_completed_job_error( - db, - &flow_job, - 0, - None, - e, - worker_name, - true, - None, - ) - .await; + let _ = add_completed_job_error(db, &flow_job, 0, None, e, worker_name, true, None) + .await; true } Ok(_) => false, @@ -1148,23 +1154,25 @@ pub async fn update_flow_status_after_job_completion_internal( if let Some(parent_job) = flow_job.parent_job { tracing::info!(subflow_id = %flow_job.id, parent_id = %parent_job, "subflow is finished, updating parent flow status"); - return Ok(Some(RecUpdateFlowStatusAfterJobCompletion { - flow: parent_job, - job_id_for_status: flow, - success: success && !is_failure_step, - result: nresult.clone(), - stop_early_override: if stop_early { - Some(skip_if_stop_early) - } else { - None + return Ok(UpdateFlowStatusAfterJobCompletion::Rec( + RecUpdateFlowStatusAfterJobCompletion { + flow: parent_job, + job_id_for_status: flow, + success: success && !is_failure_step, + result: nresult.clone(), + stop_early_override: if stop_early { + Some(skip_if_stop_early) + } else { + None + }, + skip_error_handler: skip_error_handler || is_failure_step, }, - skip_error_handler: skip_error_handler || is_failure_step, - })); + )); } } - Ok(None) + Ok(UpdateFlowStatusAfterJobCompletion::Done(flow_job)) } else { - Ok(None) + Ok(UpdateFlowStatusAfterJobCompletion::NotDone) } } @@ -1292,7 +1300,7 @@ async fn compute_skip_branchall_failure<'c>( // ))) // } -fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u16, Duration)> { +fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u32, Duration)> { (status.fail_count <= MAX_RETRY_ATTEMPTS) .then(|| &retry) .and_then(|retry| retry.interval(status.fail_count, false)) @@ -1495,8 +1503,7 @@ pub async fn handle_flow( worker_dir: &str, job_completed_tx: Sender, ) -> anyhow::Result<()> { - let flow = flow_value - .with_context(|| "Unable to parse flow definition")?; + let flow = flow_value.with_context(|| "Unable to parse flow definition")?; let status = flow_job .parse_flow_status() .with_context(|| "Unable to parse flow status")?; @@ -1511,14 +1518,15 @@ pub async fn handle_flow( let schedule_path = flow_job.schedule_path.as_ref().unwrap(); - let schedule = - get_schedule_opt(&mut tx, &flow_job.workspace_id, schedule_path).warn_after_seconds(5).await?; + let schedule = get_schedule_opt(&mut tx, &flow_job.workspace_id, schedule_path) + .warn_after_seconds(5) + .await?; tx.commit().await?; if let Some(schedule) = schedule { if let Err(err) = handle_maybe_scheduled_job( - db, + db, &flow_job, &schedule, flow_job.script_path.as_ref().unwrap(), @@ -2247,14 +2255,16 @@ async fn push_next_flow_job( } else { let value = module.get_value(); match &value { - Ok(_) if matches!(value, Ok(FlowModuleValue::Identity)) || is_skipped => serde_json::from_str( - &serde_json::to_string(&PreviousResult { - previous_result: Some(&arc_last_job_result), - }) - .unwrap(), - ) - .map(Marc::new) - .map_err(|e| error::Error::InternalErr(format!("identity: {e:#}"))), + Ok(_) if matches!(value, Ok(FlowModuleValue::Identity)) || is_skipped => { + serde_json::from_str( + &serde_json::to_string(&PreviousResult { + previous_result: Some(&arc_last_job_result), + }) + .unwrap(), + ) + .map(Marc::new) + .map_err(|e| error::Error::InternalErr(format!("identity: {e:#}"))) + } Ok( FlowModuleValue::Script { input_transforms, .. } | FlowModuleValue::RawScript { input_transforms, .. } @@ -2276,7 +2286,7 @@ async fn push_next_flow_job( ) .await .map(Marc::new) - }, + } Ok(_) => Ok(arc_flow_job_args.clone()), Err(e) => { return Err(error::Error::InternalErr(format!( @@ -2533,7 +2543,9 @@ async fn push_next_flow_job( }; tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed perms for job {i} of {len}"); - let tag = if flow_job.tag == "flow" || flow_job.tag == format!("flow-{}", flow_job.workspace_id) { + let tag = if flow_job.tag == "flow" + || flow_job.tag == format!("flow-{}", flow_job.workspace_id) + { payload_tag.tag.clone() } else { Some(flow_job.tag.clone()) @@ -2950,7 +2962,7 @@ fn payload_from_modules<'a>( if let Some(id) = modules_node { return Some(JobPayload::FlowNode { id, path: path() }); } - + add_virtual_items_if_necessary(&mut modules); let mut failure_module = failure_module.cloned(); @@ -3094,7 +3106,7 @@ async fn compute_next_flow_transform( ContinuePayload::SingleJob(payload), NextStatus::NextStep, )) - }, + } FlowModuleValue::WhileloopFlow { modules, modules_node, .. } => { // if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg let is_simple = is_simple_modules(&modules, flow.failure_module.as_ref()); @@ -3196,15 +3208,21 @@ async fn compute_next_flow_transform( .into_iter() .filter_map(|i| { let Some(payload) = payload_from_modules( - modules.clone(), modules_node, flow.failure_module.as_ref(), flow.same_worker, + modules.clone(), + modules_node, + flow.failure_module.as_ref(), + flow.same_worker, || format!("{}-{i}", status.step), || format!("{}/forloop-{i}", flow_job.script_path()), - true + true, ) else { return None; }; Some(JobPayloadWithTag { - payload, tag: None, delete_after_use, timeout: None + payload, + tag: None, + delete_after_use, + timeout: None, }) }) .collect::>(); @@ -3268,28 +3286,37 @@ async fn compute_next_flow_transform( )))?, }; - let (modules, modules_node) = match branch { BranchChosen::Default => (default, default_node), BranchChosen::Branch { branch } => branches .into_iter() .nth(branch) .map(|Branch { modules, modules_node, .. }| (modules, modules_node)) - .ok_or_else(|| Error::BadRequest(format!("Unrecognized branch for BranchOne {status_module:?}")))? + .ok_or_else(|| { + Error::BadRequest(format!( + "Unrecognized branch for BranchOne {status_module:?}" + )) + })?, }; let Some(payload) = payload_from_modules( - modules, modules_node, flow.failure_module.as_ref(), flow.same_worker, + modules, + modules_node, + flow.failure_module.as_ref(), + flow.same_worker, || status.step.to_string(), || format!("{}/branchone-{}", flow_job.script_path(), status.step), - true + true, ) else { return Ok(NextFlowTransform::EmptyInnerFlows); }; Ok(NextFlowTransform::Continue( ContinuePayload::SingleJob(JobPayloadWithTag { - payload, tag: None, delete_after_use, timeout: None + payload, + tag: None, + delete_after_use, + timeout: None, }), NextStatus::BranchChosen(branch), )) @@ -3308,15 +3335,21 @@ async fn compute_next_flow_transform( .enumerate() .filter_map(|(i, Branch { modules, modules_node, .. })| { let Some(payload) = payload_from_modules( - modules, modules_node, flow.failure_module.as_ref(), flow.same_worker, + modules, + modules_node, + flow.failure_module.as_ref(), + flow.same_worker, || format!("{}-{i}", status.step), || format!("{}/branchall-{}", flow_job.script_path(), i), - false + false, ) else { return None; }; Some(JobPayloadWithTag { - payload, tag: None, delete_after_use, timeout: None + payload, + tag: None, + delete_after_use, + timeout: None, }) }) .collect::>(); @@ -3358,20 +3391,36 @@ async fn compute_next_flow_transform( let Branch { modules, modules_node, .. } = branches .into_iter() .nth(branch_status.branch) - .ok_or_else(|| Error::BadRequest(format!("Unrecognized branch for BranchAll {status_module:?}")))?; + .ok_or_else(|| { + Error::BadRequest(format!( + "Unrecognized branch for BranchAll {status_module:?}" + )) + })?; let Some(payload) = payload_from_modules( - modules, modules_node, flow.failure_module.as_ref(), flow.same_worker, + modules, + modules_node, + flow.failure_module.as_ref(), + flow.same_worker, || format!("{}-{}", status.step, branch_status.branch), - || format!("{}/branchall-{}", flow_job.script_path(), branch_status.branch), - false + || { + format!( + "{}/branchall-{}", + flow_job.script_path(), + branch_status.branch + ) + }, + false, ) else { return Ok(NextFlowTransform::EmptyInnerFlows); }; Ok(NextFlowTransform::Continue( ContinuePayload::SingleJob(JobPayloadWithTag { - payload, tag: None, delete_after_use, timeout: None + payload, + tag: None, + delete_after_use, + timeout: None, }), NextStatus::NextBranchStep(NextBranch { status: branch_status, @@ -3404,25 +3453,25 @@ async fn next_loop_iteration( | FlowModuleValue::FlowScript { input_transforms, .. } | FlowModuleValue::Flow { input_transforms, .. } => { Some(std::mem::take(input_transforms)) - }, + } _ => None, }; return Ok(NextFlowTransform::Continue( ContinuePayload::SingleJob( - payload_from_simple_module(value, db, flow_job, module, Some(inner_path())).await? + payload_from_simple_module(value, db, flow_job, module, Some(inner_path())).await?, ), - NextStatus::NextLoopIteration { - next: ns, - simple_input_transforms, - }, + NextStatus::NextLoopIteration { next: ns, simple_input_transforms }, )); } - + let Some(payload) = payload_from_modules( - modules, modules_node, flow.failure_module.as_ref(), flow.same_worker, + modules, + modules_node, + flow.failure_module.as_ref(), + flow.same_worker, || format!("{}-{}", status.step, ns.index), inner_path, - true + true, ) else { return Ok(NextFlowTransform::EmptyInnerFlows); }; @@ -3438,7 +3487,10 @@ async fn next_loop_iteration( )) } -pub(super) fn is_simple_modules(modules: &Vec, failure_module: Option<&Box>) -> bool { +pub(super) fn is_simple_modules( + modules: &Vec, + failure_module: Option<&Box>, +) -> bool { let is_simple = modules.len() == 1 && modules[0].is_simple() && modules[0].sleep.is_none() @@ -3685,11 +3737,7 @@ fn raw_script_to_payload( } fn flow_to_payload(path: String, delete_after_use: bool) -> JobPayloadWithTag { - let payload = JobPayload::Flow { - path, - dedicated_worker: None, - apply_preprocessor: false, - }; + let payload = JobPayload::Flow { path, dedicated_worker: None, apply_preprocessor: false }; JobPayloadWithTag { payload, tag: None, delete_after_use, timeout: None } } diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index a750be85bc..82baf71030 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -4,6 +4,7 @@ use std::path::{Component, Path, PathBuf}; use async_recursion::async_recursion; use serde_json::value::RawValue; use serde_json::{json, Value}; +use sha2::Digest; use sqlx::types::Json; use uuid::Uuid; use windmill_common::error::Error; @@ -12,8 +13,9 @@ use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId}; use windmill_common::get_latest_deployed_hash_for_path; use windmill_common::jobs::JobPayload; use windmill_common::scripts::ScriptHash; -use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file}; +use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, PythonAnnotations}; use windmill_common::{ + apps::AppScriptId, error::{self, to_anyhow}, flows::{add_virtual_items_if_necessary, FlowValue}, jobs::QueuedJob, @@ -27,8 +29,10 @@ use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel}; use crate::common::OccupancyMetrics; use crate::csharp_executor::generate_nuget_lockfile; -use crate::python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}; -use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; +use crate::python_executor::{ + create_dependencies_dir, handle_python_reqs, uv_pip_compile, USE_PIP_COMPILE, USE_PIP_INSTALL, +}; +use crate::rust_executor::generate_cargo_lockfile; use crate::{ bun_executor::gen_bun_lockfile, deno_executor::generate_deno_lock, @@ -648,7 +652,7 @@ pub async fn handle_flow_dependency_job( // Compute a lite version of the flow value (`RawScript` => `FlowScript`). let mut value_lite = flow.clone(); - tx = reduce( + tx = reduce_flow( tx, &mut value_lite.modules, &job_path, @@ -1026,49 +1030,68 @@ async fn insert_flow_node<'c>( flow: Option<&Json>>, ) -> Result<(sqlx::Transaction<'c, sqlx::Postgres>, FlowNodeId)> { let hash = { - use std::hash::{DefaultHasher, Hash, Hasher}; - - let mut hasher = DefaultHasher::new(); - code.hash(&mut hasher); - lock.hash(&mut hasher); - flow.inspect(|flow| flow.get().hash(&mut hasher)); - hasher.finish() as i64 + let mut hasher = sha2::Sha256::new(); + hasher.update(code.unwrap_or(&Default::default())); + hasher.update(lock.unwrap_or(&Default::default())); + hasher.update(flow.unwrap_or(&Default::default()).get()); + format!("{:x}", hasher.finalize()) }; // Insert the flow node if it doesn't exist. let id = sqlx::query_scalar!( r#" - WITH existing AS ( - SELECT id FROM flow_node - WHERE hash = $1 AND path = $2 AND workspace_id = $3 - AND (code IS NOT DISTINCT FROM $4) - AND (lock IS NOT DISTINCT FROM $5) - AND (flow IS NOT DISTINCT FROM $6) - LIMIT 1 - ), - inserted AS ( - INSERT INTO flow_node (hash, path, workspace_id, code, lock, flow) - VALUES ($1, $2, $3, $4, $5, $6) - ON CONFLICT DO NOTHING - RETURNING id - ) - SELECT id FROM existing - UNION ALL - SELECT id FROM inserted + INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id + RETURNING id "#, - hash, path, workspace_id, - code, + hash, lock, + code, flow as Option<&Json>> ) .fetch_one(&mut *tx) - .await? - .ok_or(error::Error::InternalErr("Failed to cache".to_string()))?; + .await?; Ok((tx, FlowNodeId(id))) } +async fn insert_app_script( + db: &sqlx::Pool, + app: i64, + code: String, + lock: Option, +) -> Result { + let code_sha256 = format!("{:x}", sha2::Sha256::digest(&code)); + let hash = { + let mut hasher = sha2::Sha256::new(); + hasher.update(app.to_le_bytes()); + hasher.update(&code_sha256); + hasher.update(lock.as_ref().unwrap_or(&Default::default())); + format!("{:x}", hasher.finalize()) + }; + + // Insert the app script if it doesn't exist. + sqlx::query_scalar!( + r#" + INSERT INTO app_script (app, hash, lock, code, code_sha256) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id + RETURNING id + "#, + app, + hash, + lock, + code, + code_sha256 + ) + .fetch_one(db) + .await + .map(AppScriptId) + .map_err(Into::into) +} + async fn insert_flow_modules<'c>( mut tx: sqlx::Transaction<'c, sqlx::Postgres>, path: &str, @@ -1078,7 +1101,7 @@ async fn insert_flow_modules<'c>( modules: &mut Vec, modules_node: &mut Option, ) -> Result> { - tx = Box::pin(reduce( + tx = Box::pin(reduce_flow( tx, modules, path, @@ -1110,7 +1133,7 @@ async fn insert_flow_modules<'c>( Ok(tx) } -async fn reduce<'c>( +async fn reduce_flow<'c>( mut tx: sqlx::Transaction<'c, sqlx::Postgres>, modules: &mut Vec, path: &str, @@ -1123,7 +1146,7 @@ async fn reduce<'c>( let mut val = serde_json::from_str::(module.value.get()).map_err(|err| { Error::InternalErr(format!( - "reduce: Failed to parse flow module value: {}", + "reduce_flow: Failed to parse flow module value: {}", err )) })?; @@ -1219,6 +1242,48 @@ async fn reduce<'c>( Ok(tx) } +async fn reduce_app(db: &sqlx::Pool, value: &mut Value, app: i64) -> Result<()> { + match value { + Value::Object(object) => { + if let Some(Value::Object(script)) = object.get_mut("inlineScript") { + if script + .get("language") + .and_then(|x| x.as_str()) + .is_some_and(|x| x == "frontend") + { + return Ok(()); + } + // replace `content` with an empty string: + let Some(Value::String(code)) = script.get_mut("content").map(std::mem::take) + else { + return Err(error::Error::InternalErr( + "Missing `content` in inlineScript".to_string(), + )); + }; + // remove `lock`: + let lock = script.remove("lock").and_then(|x| match x { + Value::String(s) => Some(s), + _ => None, + }); + let id = insert_app_script(db, app, code, lock).await?; + // insert the `id` into the `script` object: + script.insert("id".to_string(), json!(id.0)); + } else { + for (_, value) in object { + Box::pin(reduce_app(db, value, app)).await?; + } + } + } + Value::Array(array) => { + for value in array { + Box::pin(reduce_app(db, value, app)).await?; + } + } + _ => {} + } + Ok(()) +} + fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool { if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative { let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content); @@ -1404,11 +1469,12 @@ pub async fn handle_app_dependency_job( .clone() .ok_or_else(|| Error::InternalErr("App Dependency requires script hash".to_owned()))? .0; - let value = sqlx::query_scalar!("SELECT value FROM app_version WHERE id = $1", id) + let record = sqlx::query!("SELECT app_id, value FROM app_version WHERE id = $1", id) .fetch_optional(db) - .await?; + .await? + .map(|record| (record.app_id, record.value)); - if let Some(value) = value { + if let Some((app_id, value)) = record { let value = lock_modules_app( value, job, @@ -1425,6 +1491,21 @@ pub async fn handle_app_dependency_job( ) .await?; + // Compute a lite version of the app value (w/ `inlineScript.{lock,code}`). + let mut value_lite = value.clone(); + reduce_app(db, &mut value_lite, app_id).await?; + if let Value::Object(object) = &mut value_lite { + object.insert("version".to_string(), json!(id)); + } + sqlx::query!( + "INSERT INTO app_version_lite (id, value) VALUES ($1, $2) + ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value", + id, + sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json>, + ) + .execute(db) + .await?; + // Re-check cancelation to ensure we don't accidentially override an app. if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id) .fetch_optional(db) @@ -1496,6 +1577,8 @@ async fn python_dep( w_id: &str, worker_dir: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + no_uv_compile: bool, + no_uv_install: bool, ) -> std::result::Result { create_dependencies_dir(job_dir).await; let req: std::result::Result = uv_pip_compile( @@ -1508,7 +1591,7 @@ async fn python_dep( worker_name, w_id, occupancy_metrics, - false, + no_uv_compile, false, ) .await; @@ -1525,7 +1608,7 @@ async fn python_dep( job_dir, worker_dir, occupancy_metrics, - false, + no_uv_install, false, ) .await; @@ -1576,28 +1659,23 @@ async fn capture_dependency_job( .join("\n") }; - python_dep( - reqs, - job_id, - mem_peak, - canceled_by, - job_dir, - db, - worker_name, - w_id, - worker_dir, - &mut Some(occupancy_metrics), - ) - .await - } - ScriptLang::Ansible => { - if raw_deps { - return Err(Error::ExecutionErr( - "Raw dependencies not supported for ansible".to_string(), - )); + let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } = + PythonAnnotations::parse(job_raw_code); + + if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL { + if let Err(e) = sqlx::query!( + r#" + INSERT INTO metrics (id, value) + VALUES ('no_uv_usage_py', $1) + "#, + serde_json::to_value("").map_err(to_anyhow)? + ) + .execute(db) + .await + { + tracing::error!("Error inserting no_uv_usage_py to db: {:?}", e); + } } - let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?; - let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default(); python_dep( reqs, @@ -1610,6 +1688,48 @@ async fn capture_dependency_job( w_id, worker_dir, &mut Some(occupancy_metrics), + no_uv_compile | no_uv, + no_uv_install | no_uv, + ) + .await + } + ScriptLang::Ansible => { + if raw_deps { + return Err(Error::ExecutionErr( + "Raw dependencies not supported for ansible".to_string(), + )); + } + let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?; + let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default(); + + if *USE_PIP_COMPILE || *USE_PIP_INSTALL { + if let Err(e) = sqlx::query!( + r#" + INSERT INTO metrics (id, value) + VALUES ('no_uv_usage_ansible', $1) + "#, + serde_json::to_value("").map_err(to_anyhow)? + ) + .execute(db) + .await + { + tracing::error!("Error inserting no_uv_usage_ansible to db: {:?}", e); + }; + } + + python_dep( + reqs, + job_id, + mem_peak, + canceled_by, + job_dir, + db, + worker_name, + w_id, + worker_dir, + &mut Some(occupancy_metrics), + false, + false, ) .await } diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 86c2e9535f..dd9d7eeba8 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.435.1"; +export const VERSION = "v1.437.1"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/cli/main.ts b/cli/main.ts index 632047531b..4e112e0646 100644 --- a/cli/main.ts +++ b/cli/main.ts @@ -60,7 +60,7 @@ export { // } // }); -export const VERSION = "1.435.1"; +export const VERSION = "1.437.1"; const command = new Command() .name("wmill") diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index a1b5daca43..35b8b8ba6a 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index 94cc160cae..44dd185d25 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools # Install UV RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv -COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun +COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/ diff --git a/flake.lock b/flake.lock index 9256fa71d5..7d7118df63 100644 --- a/flake.lock +++ b/flake.lock @@ -20,11 +20,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1732014248, - "narHash": "sha256-y/MEyuJ5oBWrWAic/14LaIr/u5E0wRVzyYsouYY3W6w=", + "lastModified": 1733759999, + "narHash": "sha256-463SNPWmz46iLzJKRzO3Q2b0Aurff3U1n0nYItxq7jU=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "23e89b7da85c3640bbc2173fe04f4bd114342367", + "rev": "a73246e2eef4c6ed172979932bc80e1404ba2d56", "type": "github" }, "original": { @@ -61,11 +61,11 @@ "nixpkgs": "nixpkgs_2" }, "locked": { - "lastModified": 1732156292, - "narHash": "sha256-XuTCME5ZausokOJ28AsIoayBVD1soscdoiKweT4VY50=", + "lastModified": 1733884434, + "narHash": "sha256-8GXR9kC07dyOIshAyfZhG11xfvBRSZzYghnZ2weOKJU=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "2d484c7a0db32f2700e253160bcd2aaa6cdca3ba", + "rev": "d0483df44ddf0fd1985f564abccbe568e020ddf2", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 2c1845d176..2f7bf7f176 100644 --- a/flake.nix +++ b/flake.nix @@ -14,7 +14,7 @@ }; buildInputs = with pkgs; [ openssl openssl.dev libxml2.dev xmlsec.dev libxslt.dev - rust-bin.stable.latest.default nodejs + rust-bin.stable.latest.default nodejs cmake postgresql pkg-config ]; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index adbe47c219..c6b28dbc19 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-components", - "version": "1.435.1", + "version": "1.437.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-components", - "version": "1.435.1", + "version": "1.437.1", "license": "AGPL-3.0", "dependencies": { "@anthropic-ai/sdk": "^0.32.1", @@ -72,7 +72,7 @@ "windmill-parser-wasm-go": "^1.429.0", "windmill-parser-wasm-php": "^1.429.0", "windmill-parser-wasm-py": "^1.429.0", - "windmill-parser-wasm-regex": "^1.429.0", + "windmill-parser-wasm-regex": "^1.435.2", "windmill-parser-wasm-rust": "^1.429.0", "windmill-parser-wasm-ts": "^1.429.0", "windmill-parser-wasm-yaml": "^1.429.0", @@ -87,10 +87,10 @@ "@hey-api/openapi-ts": "^0.43.0", "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", - "@sveltejs/adapter-static": "^3.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/adapter-static": "^3.0.6", + "@sveltejs/kit": "^2.9.0", "@sveltejs/package": "^2.2.2", - "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@sveltejs/vite-plugin-svelte": "^3.1.2", "@tailwindcss/forms": "^0.5.3", "@tailwindcss/typography": "^0.5.8", "@types/d3": "^7.4.0", @@ -130,7 +130,7 @@ "tailwindcss": "^3.4.1", "tslib": "^2.6.1", "typescript": "^5.1.3", - "vite": "^5", + "vite": "^5.4.11", "vite-plugin-circular-dependency": "^0.2.1", "vite-plugin-mkcert": "^1.17.5", "yootils": "^0.3.1" @@ -3588,9 +3588,9 @@ } }, "node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==" + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.5.0.tgz", + "integrity": "sha512-gv3ZRaISU3fjPAgNsriBRqGWQL6quFx04YMPW/zD8XMLsU32mhCCbfbO6KZFLjvYpCZ8zyDEgqsgf+PwPaM7GQ==" }, "node_modules/@jridgewell/trace-mapping": { "version": "0.3.20", @@ -3901,10 +3901,11 @@ } }, "node_modules/@polka/url": { - "version": "1.0.0-next.24", - "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.24.tgz", - "integrity": "sha512-2LuNTFBIO0m7kKIQvvPHN6UE63VjpmL9rnEEaOOaiSPbZK+zUOYIzBAWcED+3XYzhYsd/0mD57VdxAEqqV52CQ==", - "dev": true + "version": "1.0.0-next.28", + "resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.28.tgz", + "integrity": "sha512-8LduaNlMZGwdZ6qWrKlfa+2M4gahzFkprZiAt2TF8uS0qQgBizKXpXURqvTJ4WtmupWxaLqjRb2UCTe72mu+Aw==", + "dev": true, + "license": "MIT" }, "node_modules/@popperjs/core": { "version": "2.11.8", @@ -4218,32 +4219,33 @@ } }, "node_modules/@sveltejs/adapter-static": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.1.tgz", - "integrity": "sha512-6lMvf7xYEJ+oGeR5L8DFJJrowkefTK6ZgA4JiMqoClMkKq0s6yvsd3FZfCFvX1fQ0tpCD7fkuRVHsnUVgsHyNg==", + "version": "3.0.6", + "resolved": "https://registry.npmjs.org/@sveltejs/adapter-static/-/adapter-static-3.0.6.tgz", + "integrity": "sha512-MGJcesnJWj7FxDcB/GbrdYD3q24Uk0PIL4QIX149ku+hlJuj//nxUbb0HxUTpjkecWfHjVveSUnUaQWnPRXlpg==", "dev": true, "peerDependencies": { "@sveltejs/kit": "^2.0.0" } }, "node_modules/@sveltejs/kit": { - "version": "2.5.0", - "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.5.0.tgz", - "integrity": "sha512-1uyXvzC2Lu1FZa30T4y5jUAC21R309ZMRG0TPt+PPPbNUoDpy8zSmSNVWYaBWxYDqLGQ5oPNWvjvvF2IjJ1jmA==", + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/@sveltejs/kit/-/kit-2.9.0.tgz", + "integrity": "sha512-W3E7ed3ChB6kPqRs2H7tcHp+Z7oiTFC6m+lLyAQQuyXeqw6LdNuuwEUla+5VM0OGgqQD+cYD6+7Xq80vVm17Vg==", "dev": true, "hasInstallScript": true, + "license": "MIT", "dependencies": { "@types/cookie": "^0.6.0", "cookie": "^0.6.0", - "devalue": "^4.3.2", - "esm-env": "^1.0.0", - "import-meta-resolve": "^4.0.0", + "devalue": "^5.1.0", + "esm-env": "^1.2.1", + "import-meta-resolve": "^4.1.0", "kleur": "^4.1.5", "magic-string": "^0.30.5", "mrmime": "^2.0.0", "sade": "^1.8.1", "set-cookie-parser": "^2.6.0", - "sirv": "^2.0.4", + "sirv": "^3.0.0", "tiny-glob": "^0.2.9" }, "bin": { @@ -4253,9 +4255,9 @@ "node": ">=18.13" }, "peerDependencies": { - "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@sveltejs/vite-plugin-svelte": "^3.0.0 || ^4.0.0-next.1 || ^5.0.0", "svelte": "^4.0.0 || ^5.0.0-next.0", - "vite": "^5.0.3" + "vite": "^5.0.3 || ^6.0.0" } }, "node_modules/@sveltejs/package": { @@ -4281,17 +4283,17 @@ } }, "node_modules/@sveltejs/vite-plugin-svelte": { - "version": "3.0.2", - "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-3.0.2.tgz", - "integrity": "sha512-MpmF/cju2HqUls50WyTHQBZUV3ovV/Uk8k66AN2gwHogNAG8wnW8xtZDhzNBsFJJuvmq1qnzA5kE7YfMJNFv2Q==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte/-/vite-plugin-svelte-3.1.2.tgz", + "integrity": "sha512-Txsm1tJvtiYeLUVRNqxZGKR/mI+CzuIQuc2gn+YCs9rMTowpNZ2Nqt53JdL8KF9bLhAf2ruR/dr9eZCwdTriRA==", "dev": true, "dependencies": { - "@sveltejs/vite-plugin-svelte-inspector": "^2.0.0", + "@sveltejs/vite-plugin-svelte-inspector": "^2.1.0", "debug": "^4.3.4", "deepmerge": "^4.3.1", "kleur": "^4.1.5", - "magic-string": "^0.30.5", - "svelte-hmr": "^0.15.3", + "magic-string": "^0.30.10", + "svelte-hmr": "^0.16.0", "vitefu": "^0.2.5" }, "engines": { @@ -4303,9 +4305,9 @@ } }, "node_modules/@sveltejs/vite-plugin-svelte-inspector": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-2.0.0.tgz", - "integrity": "sha512-gjr9ZFg1BSlIpfZ4PRewigrvYmHWbDrq2uvvPB1AmTWKuM+dI1JXQSUu2pIrYLb/QncyiIGkFDFKTwJ0XqQZZg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/@sveltejs/vite-plugin-svelte-inspector/-/vite-plugin-svelte-inspector-2.1.0.tgz", + "integrity": "sha512-9QX28IymvBlSCqsCll5t0kQVxipsfhFFL+L2t3nTWfXnddYwxBuAEtTtlaVQpRz9c37BhJjltSeY4AJSC03SSg==", "dev": true, "dependencies": { "debug": "^4.3.4" @@ -6438,10 +6440,11 @@ } }, "node_modules/devalue": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/devalue/-/devalue-4.3.2.tgz", - "integrity": "sha512-KqFl6pOgOW+Y6wJgu80rHpo2/3H07vr8ntR9rkkFIRETewbf5GaYYcakYfiKz89K+sLsuPkQIZaXDMjUObZwWg==", - "dev": true + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/devalue/-/devalue-5.1.1.tgz", + "integrity": "sha512-maua5KUiapvEwiEAe+XnlZ3Rh0GD+qI1J/nb9vrJc3muPXvcF/8gXYTWF76+5DAqHyDUtOIImEuo0YKE9mshVw==", + "dev": true, + "license": "MIT" }, "node_modules/devlop": { "version": "1.1.0", @@ -6994,9 +6997,10 @@ } }, "node_modules/esm-env": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.0.0.tgz", - "integrity": "sha512-Cf6VksWPsTuW01vU9Mk/3vRue91Zevka5SjyNf3nEpokFRuqt/KjUQoGAwq9qMmhpLTHmXzSIrFRw8zxWzmFBA==" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/esm-env/-/esm-env-1.2.1.tgz", + "integrity": "sha512-U9JedYYjCnadUlXk7e1Kr+aENQhtUaoaV9+gZm1T8LC/YBAPJx3NSPIAurFOC0U5vrdSevnUJS2/wUVxGwPhng==", + "license": "MIT" }, "node_modules/esm-env-robust": { "version": "0.0.3", @@ -7936,10 +7940,11 @@ } }, "node_modules/import-meta-resolve": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.0.0.tgz", - "integrity": "sha512-okYUR7ZQPH+efeuMJGlq4f8ubUgO50kByRPyt/Cy1Io4PSRsPjxME+YlVaCOx+NIToW7hCsZNFJyTPFFKepRSA==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/import-meta-resolve/-/import-meta-resolve-4.1.0.tgz", + "integrity": "sha512-I6fiaX09Xivtk+THaMfAwnA3MVA5Big1WHF1Dfx9hFuvNIWpXnorlkzhcQf6ehrqQiiZECRt1poOAkPmer3ruw==", "dev": true, + "license": "MIT", "funding": { "type": "github", "url": "https://github.com/sponsors/wooorm" @@ -8612,14 +8617,11 @@ } }, "node_modules/magic-string": { - "version": "0.30.5", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.5.tgz", - "integrity": "sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==", + "version": "0.30.14", + "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.14.tgz", + "integrity": "sha512-5c99P1WKTed11ZC0HMJOj6CDIue6F8ySu+bJL+85q1zBEIY8IklrJ1eiKC2NDRh3Ct3FcvmJPyQHb9erXMTJNw==", "dependencies": { - "@jridgewell/sourcemap-codec": "^1.4.15" - }, - "engines": { - "node": ">=12" + "@jridgewell/sourcemap-codec": "^1.5.0" } }, "node_modules/map-obj": { @@ -9785,6 +9787,7 @@ "resolved": "https://registry.npmjs.org/mrmime/-/mrmime-2.0.0.tgz", "integrity": "sha512-eu38+hdgojoyq63s+yTpN4XMBdt5l8HhMhc4VKLO9KM5caLIBvUm4thi7fFaxyTmCKeNnXZ5pAlBwCUnhA09uw==", "dev": true, + "license": "MIT", "engines": { "node": ">=10" } @@ -11964,17 +11967,18 @@ "dev": true }, "node_modules/sirv": { - "version": "2.0.4", - "resolved": "https://registry.npmjs.org/sirv/-/sirv-2.0.4.tgz", - "integrity": "sha512-94Bdh3cC2PKrbgSOUqTiGPWVZeSiXfKOVZNJniWoqrWrRkB1CJzBU3NEbiTsPcYy1lDsANA/THzS+9WBiy5nfQ==", + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/sirv/-/sirv-3.0.0.tgz", + "integrity": "sha512-BPwJGUeDaDCHihkORDchNyyTvWFhcusy1XMmhEVTQTwGeybFbp8YEmB+njbPnth1FibULBSBVwCQni25XlCUDg==", "dev": true, + "license": "MIT", "dependencies": { "@polka/url": "^1.0.0-next.24", "mrmime": "^2.0.0", "totalist": "^3.0.0" }, "engines": { - "node": ">= 10" + "node": ">=18" } }, "node_modules/slash": { @@ -12646,9 +12650,9 @@ } }, "node_modules/svelte-hmr": { - "version": "0.15.3", - "resolved": "https://registry.npmjs.org/svelte-hmr/-/svelte-hmr-0.15.3.tgz", - "integrity": "sha512-41snaPswvSf8TJUhlkoJBekRrABDXDMdpNpT2tfHIv4JuhgvHqLMhEPGtaQn0BmbNSTkuz2Ed20DF2eHw0SmBQ==", + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/svelte-hmr/-/svelte-hmr-0.16.0.tgz", + "integrity": "sha512-Gyc7cOS3VJzLlfj7wKS0ZnzDVdv3Pn2IuVeJPk9m2skfhcu5bq3wtIZyQGggr7/Iim5rH5cncyQft/kRLupcnA==", "dev": true, "engines": { "node": "^12.20 || ^14.13.1 || >= 16" @@ -13068,6 +13072,7 @@ "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz", "integrity": "sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==", "dev": true, + "license": "MIT", "engines": { "node": ">=6" } @@ -13399,15 +13404,14 @@ } }, "node_modules/vite": { - "version": "5.4.0", - "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.0.tgz", - "integrity": "sha512-5xokfMX0PIiwCMCMb9ZJcMyh5wbBun0zUzKib+L65vAZ8GY9ePZMXxFrHbr/Kyll2+LSCY7xtERPpxkBDKngwg==", + "version": "5.4.11", + "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.11.tgz", + "integrity": "sha512-c7jFQRklXua0mTzneGW9QVyxFjUgwcihC4bXEtujIo2ouWCe1Ajt/amn2PCxYnhYfd5k09JX3SB7OYWFKYqj8Q==", "dev": true, - "license": "MIT", "dependencies": { "esbuild": "^0.21.3", - "postcss": "^8.4.40", - "rollup": "^4.13.0" + "postcss": "^8.4.43", + "rollup": "^4.20.0" }, "bin": { "vite": "bin/vite.js" @@ -13660,9 +13664,9 @@ "integrity": "sha512-cqc+tblQVHVrc8wNA4esVYD1Dv59XQJ4mHXFFPa8Lx5UjXv7FO/0VQxyQuRyXaP/V6J6DDy0oeN107eFNLxrBg==" }, "node_modules/windmill-parser-wasm-regex": { - "version": "1.429.0", - "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.429.0.tgz", - "integrity": "sha512-rjgrgSYjEBU2+RB6ffFR+nmvEJdiqVKYiwugzHQE+VgVTYMJ0jLBE370n5dBjDj5KPsF+Sfmj6DIUK69uu9wSA==" + "version": "1.435.2", + "resolved": "https://registry.npmjs.org/windmill-parser-wasm-regex/-/windmill-parser-wasm-regex-1.435.2.tgz", + "integrity": "sha512-kQShKPNjXok8jV7b7llfvKhypHfqePwiKgTHm8MlzBt4BZC87xrQ7TH0eS9h7OSK9Q98H9fIiUr2AbN8zTCraA==" }, "node_modules/windmill-parser-wasm-rust": { "version": "1.429.0", diff --git a/frontend/package.json b/frontend/package.json index 260d45e079..7427139167 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "windmill-components", - "version": "1.435.1", + "version": "1.437.1", "scripts": { "dev": "vite dev", "build": "vite build", @@ -21,10 +21,10 @@ "@hey-api/openapi-ts": "^0.43.0", "@playwright/test": "^1.34.3", "@rgossiaux/svelte-headlessui": "^2.0.0", - "@sveltejs/adapter-static": "^3.0.0", - "@sveltejs/kit": "^2.0.0", + "@sveltejs/adapter-static": "^3.0.6", + "@sveltejs/kit": "^2.9.0", "@sveltejs/package": "^2.2.2", - "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@sveltejs/vite-plugin-svelte": "^3.1.2", "@tailwindcss/forms": "^0.5.3", "@tailwindcss/typography": "^0.5.8", "@types/d3": "^7.4.0", @@ -64,7 +64,7 @@ "tailwindcss": "^3.4.1", "tslib": "^2.6.1", "typescript": "^5.1.3", - "vite": "^5", + "vite": "^5.4.11", "vite-plugin-circular-dependency": "^0.2.1", "vite-plugin-mkcert": "^1.17.5", "yootils": "^0.3.1" @@ -128,8 +128,8 @@ "ol": "^7.4.0", "openai": "^4.57.2", "p-limit": "^6.1.0", - "pdfjs-dist": "4.8.69", "panzoom": "^9.4.3", + "pdfjs-dist": "4.8.69", "quill": "^1.3.7", "rfc4648": "^1.5.3", "svelte-carousel": "^1.0.25", @@ -145,7 +145,7 @@ "windmill-parser-wasm-go": "^1.429.0", "windmill-parser-wasm-php": "^1.429.0", "windmill-parser-wasm-py": "^1.429.0", - "windmill-parser-wasm-regex": "^1.429.0", + "windmill-parser-wasm-regex": "^1.435.2", "windmill-parser-wasm-rust": "^1.429.0", "windmill-parser-wasm-ts": "^1.429.0", "windmill-parser-wasm-yaml": "^1.429.0", diff --git a/frontend/src/app.html b/frontend/src/app.html index 7f93534ddb..718800105a 100644 --- a/frontend/src/app.html +++ b/frontend/src/app.html @@ -27,61 +27,63 @@
-
-
+
+
+
-
- - - - - + + + + + - - - - - -
- Loading... + /> + + + + + +
+ Loading... +
%sveltekit.body% diff --git a/frontend/src/lib/components/ArgInput.svelte b/frontend/src/lib/components/ArgInput.svelte index e19a48967e..9edfb332de 100644 --- a/frontend/src/lib/components/ArgInput.svelte +++ b/frontend/src/lib/components/ArgInput.svelte @@ -132,9 +132,8 @@ let el: HTMLTextAreaElement | undefined = undefined - let inputCat = computeInputCat(type, format, itemsType?.type, enum_, contentEncoding) - $: inputCat = computeInputCat(type, format, itemsType?.type, enum_, contentEncoding) + let rawValue: string | undefined = undefined function computeDefaultValue( @@ -211,12 +210,12 @@ $: defaultValue != undefined && handleDefaultValueChange() - let oldDefaultValue = defaultValue + let oldDefaultValue = structuredClone(defaultValue) function handleDefaultValueChange() { - if (value == oldDefaultValue) { + if (deepEqual(value, oldDefaultValue)) { value = defaultValue } - oldDefaultValue = defaultValue + oldDefaultValue = structuredClone(defaultValue) } function evalValueToRaw() { @@ -497,7 +496,6 @@ {onlyMaskPassword} {disablePortal} {disabled} - noDelete schema={getSchemaFromProperties(itemsType?.properties)} bind:args={v} /> @@ -705,7 +703,6 @@ {onlyMaskPassword} {disablePortal} {disabled} - noDelete schema={{ properties: Object.fromEntries( Object.entries(obj.properties).filter(([k, v]) => k !== 'label') diff --git a/frontend/src/lib/components/AuthSettings.svelte b/frontend/src/lib/components/AuthSettings.svelte new file mode 100644 index 0000000000..827f5cd770 --- /dev/null +++ b/frontend/src/lib/components/AuthSettings.svelte @@ -0,0 +1,256 @@ + + +
+ + SSO + OAuth + SCIM/SAML + +
+ +
+ {#if tab === 'sso'} + {#if !$enterpriseLicense || $enterpriseLicense.endsWith('_pro')} + + Without EE, the number of SSO users is limited to 10. SCIM/SAML is available on EE + + {/if} + +
+
+ When at least one of the below options is set, users will be able to login to Windmill via + their third-party account. +
To test SSO, the recommended workflow is to to save the settings and try to login in + an incognito window. + Learn more +
+
+ + + + + + + + + + + + {#each Object.keys(oauths) as k} + {#if !['authelia', 'authentik', 'google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'keycloak', 'slack', 'kanidm', 'zitadel'].includes(k) && 'login_config' in oauths[k]} + {#if oauths[k]} +
+
+ + + { + delete oauths[k] + oauths = { ...oauths } + }} + /> +
+
+ + + + {#if !windmillBuiltins.includes(k) && k != 'slack'} + + {/if} +
+
+ {/if} + {/if} + {/each} +
+
+ + +
+
+ +
+ {:else if tab === 'oauth'} +
+ When one of the below options is set, you will be able to create a specific resource + containing a token automatically generated by the third-party provider. +
+ To test it after setting an oauth client, go to the Resources menu and create a new one of the + type of your oauth client (i.e. a 'github' resource if you set Github OAuth). +
Learn more +
+
+ +
+ + {#each Object.keys(oauths) as k} + {#if oauths[k] && !('login_config' in oauths[k])} + {#if !['slack'].includes(k) && oauths[k]} +
+
+ + + { + delete oauths[k] + oauths = { ...oauths } + }} + /> +
+
+ + + {#if !windmillBuiltins.includes(k) && k != 'slack'} + + {/if} + {#if k == 'snowflake_oauth'} + + {/if} +
+
+ {/if} + {/if} + {/each} + +
+ + {#if oauth_name == 'custom'} + + {:else} + + {/if} + +
+ {:else if tab == 'scim'} + + {/if} +
diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 27818a6c54..f0fd68c310 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -211,7 +211,6 @@ export let folding = false export let args: Record | undefined = undefined export let useWebsockets: boolean = true - export let listenEmptyChanges = false export let small = false export let scriptLang: Preview['language'] | 'bunnative' export let disabled: boolean = false @@ -414,6 +413,10 @@ } } + export function getScriptLang(): string | undefined { + return scriptLang + } + let command: Disposable | undefined = undefined let sqlTypeCompletor: Disposable | undefined = undefined @@ -1148,10 +1151,8 @@ timeoutModel && clearTimeout(timeoutModel) timeoutModel = setTimeout(() => { let ncode = getCode() - if (ncode != '' || listenEmptyChanges) { - code = ncode - dispatch('change', code) - } + code = ncode + dispatch('change', code) }, 500) ataModel && clearTimeout(ataModel) diff --git a/frontend/src/lib/components/FlowGraphViewer.svelte b/frontend/src/lib/components/FlowGraphViewer.svelte index 326e1275b6..f69c1c9450 100644 --- a/frontend/src/lib/components/FlowGraphViewer.svelte +++ b/frontend/src/lib/components/FlowGraphViewer.svelte @@ -36,6 +36,8 @@ > {#if notAnonynmous} @@ -947,7 +948,12 @@
{#each flowJobIds?.flowJobs ?? [] as loopJobId, j (loopJobId)} - {#if render} + {#if render && j + subflowsSize + 1 == (flowJobIds?.flowJobs.length ?? 0)} + + {/if} + {#if render && j + subflowsSize + 1 > (flowJobIds?.flowJobs.length ?? 0)}