starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-04 08:01:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into win-444-add-support-for-csharp-c

Browse Source
This commit is contained in:
wendrul
2024-12-12 11:09:24 +01:00
parent dc055604d8 0dab23f6bf
commit e66ee4f5a1
163 changed files with 5727 additions and 3251 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/change-versions-mac.sh
+1 -1
View File
@@ -16,7 +16,7 @@ sed -i '' -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/
sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i '' -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i '' -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i '' -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
sed -i '' -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
# sed -i '' -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
sed -i '' -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
sed -i '' -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
.github/change-versions.sh
+1 -1
View File
@@ -17,7 +17,7 @@ sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" ${root_dirpath}/frontend/pac
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill/pyproject.toml
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i -e "/^ModuleVersion =/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
sed -i -e "/^[[:space:]]*ModuleVersion[[:space:]]*=/s/= .*/= '$VERSION'/" ${root_dirpath}/powershell-client/WindmillClient/WindmillClient.psd1
# sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" ${root_dirpath}/python-client/wmill_pg/pyproject.toml
sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" ${root_dirpath}/lsp/Pipfile
.github/workflows/build-publish-rh-image.yml
+2 -2
View File
@@ -64,7 +64,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
@@ -81,7 +81,7 @@ jobs:
platforms: linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql
secrets: |
rh_username=${{ secrets.RH_USERNAME }}
rh_password=${{ secrets.RH_PASSWORD }}
.github/workflows/build-staging-image.yml
+1 -1
View File
@@ -62,7 +62,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,deno_core,kafka,php,mysql
tags: |
${{ steps.meta-ee-public.outputs.tags }}
labels: |
.github/workflows/build_windows_worker_.yml
+1 -1
View File
@@ -45,7 +45,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql
- name: Rename binary with corresponding architecture
run: |
.github/workflows/docker-image-rpi4.yml
+1 -1
View File
@@ -67,7 +67,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,deno_core
features=embedding,parquet,openidconnect,deno_core,php,mysql
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:dev
${{ steps.meta-public.outputs.tags }}
.github/workflows/docker-image.yml
+3 -3
View File
@@ -76,7 +76,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,jemalloc,deno_core
features=embedding,parquet,openidconnect,jemalloc,deno_core,dind,php,mysql
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
${{ steps.meta-public.outputs.tags }}
@@ -138,7 +138,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
${{ steps.meta-ee-public.outputs.tags }}
@@ -200,7 +200,7 @@ jobs:
platforms: linux/amd64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,otel,dind,php,mysql
PYTHON_IMAGE=python:3.12.2-slim-bookworm
tags: |
${{ steps.meta-ee-public-py312.outputs.tags }}
.github/workflows/publish_windows_worker.yml
+1 -1
View File
@@ -47,7 +47,7 @@ jobs:
$env:OPENSSL_DIR="${Env:VCPKG_INSTALLATION_ROOT}\installed\x64-windows-static"
mkdir frontend/build && cd backend
New-Item -Path . -Name "windmill-api/openapi-deref.yaml" -ItemType "File" -Force
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka
cargo build --release --features=enterprise,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,deno_core,kafka,php,mysql
- name: Rename binary with corresponding architecture
run: |
CHANGELOG.md
+44
View File
@@ -1,5 +1,49 @@
# Changelog
## [1.437.1](https://github.com/windmill-labs/windmill/compare/v1.437.0...v1.437.1) (2024-12-10)
### Bug Fixes
* **frontend:** fix newly deployed apps with frontend scripts ([bcd2cfe](https://github.com/windmill-labs/windmill/commit/bcd2cfe674038042f08f22d05286596b3442caf9))
## [1.437.0](https://github.com/windmill-labs/windmill/compare/v1.436.0...v1.437.0) (2024-12-10)
### Features
* add ctx.name to apps [#4885](https://github.com/windmill-labs/windmill/issues/4885) ([2387503](https://github.com/windmill-labs/windmill/commit/23875033dd19d6d585f2433fa7b18a703175e329))
### Bug Fixes
* **parsers:** more robust pwsh param parsing ([#4884](https://github.com/windmill-labs/windmill/issues/4884)) ([7014389](https://github.com/windmill-labs/windmill/commit/7014389d677fba563bec02e08815452b195ecd0a))
* **python:** do not follow symlinks when copying recursively in overlapping sites-package ([c173e46](https://github.com/windmill-labs/windmill/commit/c173e46a724b50d27416bbc169bf2a57fe68808c))
## [1.436.0](https://github.com/windmill-labs/windmill/compare/v1.435.2...v1.436.0) (2024-12-09)
### Features
* add db storage for app inline scripts ([#4837](https://github.com/windmill-labs/windmill/issues/4837)) ([2bc4934](https://github.com/windmill-labs/windmill/commit/2bc4934c4f38ca2631eb99140d89a0b3d3b3a2cf))
* add otlp support ([#4869](https://github.com/windmill-labs/windmill/issues/4869)) ([6d04744](https://github.com/windmill-labs/windmill/commit/6d047449e200d51f060d4107ff5ad0e10af99a66))
* **cache:** refurbish fs backed cache ([#4863](https://github.com/windmill-labs/windmill/issues/4863)) ([3c4408e](https://github.com/windmill-labs/windmill/commit/3c4408e3dbf8a37c0b862805b1991af3d39ac054))
* docker runtime ([2c5d07f](https://github.com/windmill-labs/windmill/commit/2c5d07f3d9069b21b877ea2d085f0a851b1975c1))
* **frontend:** render new job kinds ([#4864](https://github.com/windmill-labs/windmill/issues/4864)) ([691ef64](https://github.com/windmill-labs/windmill/commit/691ef6468823c99ab129c53ae699de1a91f77c97))
### Bug Fixes
* fix `flow_node` uniqueness ([#4850](https://github.com/windmill-labs/windmill/issues/4850)) ([667167a](https://github.com/windmill-labs/windmill/commit/667167a022ed31cb67a6c189cc5bd54b09473f4f))
* handle flow & workspace renames for `flow_node` ([#4861](https://github.com/windmill-labs/windmill/issues/4861)) ([f175158](https://github.com/windmill-labs/windmill/commit/f175158b9ffcc0d9c35938e61dfe3ce56d3b64c2))
## [1.435.2](https://github.com/windmill-labs/windmill/compare/v1.435.1...v1.435.2) (2024-12-05)
### Bug Fixes
* job search toast on error ([#4851](https://github.com/windmill-labs/windmill/issues/4851)) ([a99e63f](https://github.com/windmill-labs/windmill/commit/a99e63f5435725c42e38b46933ff3caa345002b5))
## [1.435.1](https://github.com/windmill-labs/windmill/compare/v1.435.0...v1.435.1) (2024-12-05)
Dockerfile
+2 -2
View File
@@ -175,9 +175,9 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
COPY --from=builder /frontend/build /static_frontend
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=denoland/deno:2.0.4 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=denoland/deno:2.1.2 --chmod=755 /usr/bin/deno /usr/bin/deno
COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun
COPY --from=php:8.3.7-cli /usr/local/bin/php /usr/bin/php
COPY --from=composer:2.7.6 /usr/bin/composer /usr/bin/composer
backend/.sqlx/query-0ad36c1598ff4ece0c325eaeb9a9177a87e1accd192402e21db5ae09c3498ab0.json
Generated
+2 -1
View File
@@ -44,7 +44,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO app_script (app, hash, lock, code, code_sha256)\n VALUES ($1, $2, $3, $4, $5)\n ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8",
"Bpchar",
"Text",
"Text",
"Bpchar"
]
},
"nullable": [
false
]
},
"hash": "0c6c80746733be8f561ab0b631854799f5e8122adaf35465cb16c3dc795bdc3b"
}
backend/.sqlx/query-1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT format('rawscript/%s', code_sha256) as \"path!: String\"\n FROM app_script WHERE id = $1 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "path!: String",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
null
]
},
"hash": "1bae415f9440cc1334f24ce3009242cf3a6287e7b4548c7f01ad888230c27013"
}
backend/.sqlx/query-240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0.json
Generated
+32
View File
@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT \n MAX (created_at) AS last_deploy, \n COUNT (*) AS deploys_count \n , 'python' AS language\n FROM metrics \n WHERE id = 'no_uv_usage_py'\n\n UNION ALL\n \n SELECT \n MAX (created_at) AS last_deploy, \n COUNT (*) AS deploys_count \n , 'ansible' AS language\n FROM metrics \n WHERE id = 'no_uv_usage_ansible'\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "last_deploy",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "deploys_count",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "language",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null,
null
]
},
"hash": "240cf4ba63ec39a7ccfa8360824259d2fdc6681bbbd44e3ccde0a3893f6cf9a0"
}
backend/.sqlx/query-337f31c2172194cd594042c561998a03f751b246c40daf056fced0fd91f6dd73.json
Generated
+2 -1
View File
@@ -34,7 +34,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d.json
Generated
+27
View File
@@ -0,0 +1,27 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Bpchar",
"Text",
"Text",
"Jsonb"
]
},
"nullable": [
false
]
},
"hash": "5af51d5bf7614274ade044120045893edb73694601544fea7cf20f45dd25a88d"
}
backend/.sqlx/query-5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT policy as \"policy: sqlx::types::Json<Box<RawValue>>\"\n FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "policy: sqlx::types::Json<Box<RawValue>>",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Text",
"Text"
]
},
"nullable": [
false
]
},
"hash": "5d7081a9ba0d702f63ed9d44be5ffe0ba043565790865c6a6f52fab6ce340d2c"
}
backend/.sqlx/query-a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7.json → backend/.sqlx/query-6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973.json
Generated
+8 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at",
"query": "SELECT worker, worker_instance, vcpus, memory, ping_at, started_at, worker_group FROM worker_ping WHERE ping_at > now() - interval '30 days' ORDER BY started_at",
"describe": {
"columns": [
{
@@ -32,6 +32,11 @@
"ordinal": 5,
"name": "started_at",
"type_info": "Timestamptz"
},
{
"ordinal": 6,
"name": "worker_group",
"type_info": "Varchar"
}
],
"parameters": {
@@ -43,8 +48,9 @@
true,
true,
false,
false,
false
]
},
"hash": "a51007ca7b509b92faa6fe7aa59fa738594a2b7eaab8e38f3ca30e548f9a49a7"
"hash": "6a72df33cf12824c54b29dbf011f2390af9d02efc7112a32a84e1a1247a95973"
}
backend/.sqlx/query-3d363466d79075df3f74f946eff43ca89faefca3bcdf2c533425ca3868b0369a.json → backend/.sqlx/query-6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b.json
Generated
+14 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT * FROM usr where username = $1 AND workspace_id = $2",
"query": "SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2\n ",
"describe": {
"columns": [
{
@@ -42,6 +42,16 @@
"ordinal": 7,
"name": "role",
"type_info": "Varchar"
},
{
"ordinal": 8,
"name": "super_admin",
"type_info": "Bool"
},
{
"ordinal": 9,
"name": "name",
"type_info": "Varchar"
}
],
"parameters": {
@@ -58,8 +68,10 @@
false,
false,
false,
true,
false,
true
]
},
"hash": "3d363466d79075df3f74f946eff43ca89faefca3bcdf2c533425ca3868b0369a"
"hash": "6aabe704395c9be30c86d15a5d22f3509b4fcea56227b019588837132b64d58b"
}
backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json
Generated
+2 -1
View File
@@ -28,7 +28,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_py', $1)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "78cd3f9d43dcf292cfa97ed79f9b6ad60469d5a4949729676abdefb3ab2b1a7f"
}
backend/.sqlx/query-8bd028c8b5f8a4d566f89eebc2e63fd04beaf2b0b49e07c7df42ecddd70737f3.json
Generated
+18
View File
@@ -0,0 +1,18 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO resource\n (workspace_id, path, value, resource_type, created_by, edited_at)\n VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path)\n DO UPDATE SET value = $3, edited_at = now()",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Varchar",
"Jsonb",
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "8bd028c8b5f8a4d566f89eebc2e63fd04beaf2b0b49e07c7df42ecddd70737f3"
}
backend/.sqlx/query-9d3556319411a27a875bf6cf0e5eda837cc63e4d8be912c0b5bfeea4a0c8db2e.json
Generated
+2 -1
View File
@@ -28,7 +28,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052.json
Generated
+14
View File
@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO metrics (id, value) \n VALUES ('no_uv_usage_ansible', $1)\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "d4878143442a303e624ab78300464a77334a0aad0cbf219250e4e811b1d16052"
}
backend/.sqlx/query-dc32e9e81eab9cda88875020acc414271b96d251bcf54c153c45370d917eea9d.json
Generated
-27
View File
@@ -1,27 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n WITH existing AS (\n SELECT id FROM flow_node\n WHERE hash = $1 AND path = $2 AND workspace_id = $3\n AND (code IS NOT DISTINCT FROM $4)\n AND (lock IS NOT DISTINCT FROM $5)\n AND (flow IS NOT DISTINCT FROM $6)\n LIMIT 1\n ),\n inserted AS (\n INSERT INTO flow_node (hash, path, workspace_id, code, lock, flow)\n VALUES ($1, $2, $3, $4, $5, $6)\n ON CONFLICT DO NOTHING\n RETURNING id\n )\n SELECT id FROM existing\n UNION ALL\n SELECT id FROM inserted\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8",
"Text",
"Text",
"Text",
"Text",
"Jsonb"
]
},
"nullable": [
null
]
},
"hash": "dc32e9e81eab9cda88875020acc414271b96d251bcf54c153c45370d917eea9d"
}
backend/.sqlx/query-29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a.json → backend/.sqlx/query-ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b.json
Generated
+8 -2
View File
@@ -1,10 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value FROM app_version WHERE id = $1",
"query": "SELECT app_id, value FROM app_version WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "app_id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "value",
"type_info": "Json"
}
@@ -15,8 +20,9 @@
]
},
"nullable": [
false,
false
]
},
"hash": "29fce7e56bb4a874b4c1e3a6454273203aab7344abd03df4d86e4c5f5b9a394a"
"hash": "ea9bbb972217bab4d7e8f4c08e331899161e80c239d56eb657d73bbf4272939b"
}
backend/.sqlx/query-ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO app_version_lite (id, value) VALUES ($1, $2)\n ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Jsonb"
]
},
"nullable": []
},
"hash": "ee16199b4af456198fae062e948914fca6fc0a8787e4f8d520766b1c89f23602"
}
backend/.sqlx/query-f9fc0084fe086ef80005bb64a8bb6b493e53583017c18e2ab44f44125c52d548.json
Generated
+2 -1
View File
@@ -48,7 +48,8 @@
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode"
"flownode",
"appscript"
]
}
}
backend/.sqlx/query-ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c.json
Generated
+28
View File
@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "lock",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "code",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
true,
false
]
},
"hash": "ffa86babfcab107caffb8dda31a66b142fa628b8983b966357deb3d5e0a1df3c"
}
backend/Cargo.lock
Generated
+483 -179
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+28 -13
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.435.1"
version = "1.437.1"
authors.workspace = true
edition.workspace = true
@@ -30,7 +30,7 @@ members = [
]
[workspace.package]
version = "1.435.1"
version = "1.437.1"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -64,6 +64,10 @@ tantivy = ["dep:windmill-indexer", "windmill-api/tantivy"]
sqlx = ["windmill-worker/sqlx"]
deno_core = ["windmill-worker/deno_core", "dep:deno_core"]
kafka = ["windmill-api/kafka"]
otel = ["windmill-common/otel", "windmill-worker/otel"]
dind = ["windmill-worker/dind"]
php = ["windmill-worker/php"]
mysql = ["windmill-worker/mysql"]
[dependencies]
anyhow.workspace = true
@@ -96,7 +100,6 @@ deno_core = { workspace = true, optional = true }
object_store = { workspace = true, optional = true }
quote.workspace = true
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemallocator = { optional = true, workspace = true }
tikv-jemalloc-sys = { optional = true, workspace = true }
@@ -139,7 +142,7 @@ windmill-api-client = { path = "./windmill-api-client" }
axum = { version = "^0.7", features = ["multipart"] }
headers = "^0"
hyper = { version = "^1", features = ["full"] }
tokio = { version = "^1", features = ["full", "tracing"] }
tokio = { version = "^1.42.0", features = ["full", "tracing"] }
tower = "^0"
tower-http = { version = "^0.6", features = ["trace", "cors"] }
tower-cookies = "^0.10"
@@ -184,15 +187,16 @@ itertools = "^0"
regex = "^1"
semver = "^1"
deno_fetch = "0.195.0"
deno_tls = "0.158.0"
deno_console = "0.171.0"
deno_url = "0.171.0"
deno_webidl = "0.171.0"
deno_web = "0.202.0"
deno_net = "0.163.0"
deno_core = "0.311.0"
deno_ast = { version = "=0.42.2", features = ["transpiling"] }
deno_fetch = "0.203.0"
deno_tls = "0.166.0"
deno_console = "0.179.0"
deno_url = "0.179.0"
deno_webidl = "0.179.0"
deno_web = "0.210.0"
deno_net = "0.171.0"
deno_core = "0.321.0"
deno_ast = { version = "=0.43.3", features = ["transpiling"] }
deno_permissions = "0.39.0"
swc_common = "=0.37.5"
swc_ecma_parser = "=0.149.1"
@@ -281,6 +285,17 @@ tar = "^0"
http = "^1"
async-stream = "^0"
opentelemetry = "0.27.0"
tracing-opentelemetry = "0.28.0"
opentelemetry_sdk = { version = "*", features = ["rt-tokio"] }
opentelemetry-otlp = { version = "0.27.0", features = ["grpc-tonic", "tls"] }
opentelemetry-appender-tracing = "0.27.0"
opentelemetry-semantic-conventions = { version = "*", features = ["semconv_experimental"] }
bollard = "0.18.1"
tonic = { version = "^0", features = ["tls-native-roots"] }
tikv-jemallocator = { version = "0.5" }
tikv-jemalloc-sys = { version = "^0.5" }
tikv-jemalloc-ctl = { version = "^0.5" }
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
8606d98a692d11b09a387c5efbd6b4335c533fd3
89a221f8b6e0dc431c668ed804066c97b428f6ef
backend/migrations/20241202095902_app_script.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add down migration script here
DROP TABLE IF EXISTS app_version_lite;
DROP TABLE IF EXISTS app_script;
backend/migrations/20241202095902_app_script.up.sql
+26
View File
@@ -0,0 +1,26 @@
-- Add up migration script here
ALTER TYPE JOB_KIND ADD VALUE IF NOT EXISTS 'appscript';
-- Same as `app_version` but with a "lite" value (w/ `inlineScript.{code,lock}`).
CREATE TABLE app_version_lite (
id BIGSERIAL PRIMARY KEY,
value JSONB,
FOREIGN KEY (id) REFERENCES app_version (id) ON DELETE CASCADE
);
GRANT ALL ON app_version_lite TO windmill_user;
GRANT ALL ON app_version_lite TO windmill_admin;
-- App `inlineScript`.
CREATE TABLE app_script (
id BIGSERIAL PRIMARY KEY,
app BIGSERIAL NOT NULL,
hash CHAR(64) NOT NULL UNIQUE, -- sha256 of `app`, `lock`, `code`.
lock TEXT,
code TEXT NOT NULL,
code_sha256 CHAR(64) NOT NULL, -- used to retrieve the policy.
FOREIGN KEY (app) REFERENCES app (id) ON DELETE CASCADE
);
GRANT ALL ON app_script TO windmill_user;
GRANT ALL ON app_script TO windmill_admin;
backend/migrations/20241205135747_fix_flow_node_uniqueness.down.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add down migration script here
ALTER TABLE flow_node DROP COLUMN hash_v2;
backend/migrations/20241205135747_fix_flow_node_uniqueness.up.sql
+4
View File
@@ -0,0 +1,4 @@
-- Add up migration script here
CREATE SEQUENCE IF NOT EXISTS flow_node_hash_seq;
ALTER TABLE flow_node ALTER COLUMN hash DROP NOT NULL;
ALTER TABLE flow_node ADD COLUMN hash_v2 CHAR(64) NOT NULL UNIQUE DEFAULT to_hex(nextval('flow_node_hash_seq'));
backend/migrations/20241206075559_flow_node_unique_2.down.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add down migration script here
ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_unique_2;
ALTER TABLE flow_node ADD CONSTRAINT flow_node_hash_v2_key UNIQUE (hash_v2);
backend/migrations/20241206075559_flow_node_unique_2.up.sql
+3
View File
@@ -0,0 +1,3 @@
-- Add up migration script here
ALTER TABLE flow_node ADD CONSTRAINT flow_node_unique_2 UNIQUE (path, workspace_id, hash_v2);
ALTER TABLE flow_node DROP CONSTRAINT IF EXISTS flow_node_hash_v2_key;
backend/migrations/20241210184049_drop_app_lite.down.sql
+1
View File
@@ -0,0 +1 @@
-- Add down migration script here
backend/migrations/20241210184049_drop_app_lite.up.sql
+2
View File
@@ -0,0 +1,2 @@
-- Add up migration script here
delete from app_version_lite;
backend/oauth_connect.json
+8
View File
@@ -172,5 +172,13 @@
"user-library-read"
]
},
"xero": {
"auth_url": "https://login.xero.com/identity/connect/authorize",
"token_url": "https://identity.xero.com/connect/token",
"scopes": [
"offline_access",
"accounting.transactions"
]
},
"snowflake_oauth": {}
}
backend/parsers/windmill-parser-bash/src/lib.rs
+156 -25
View File
@@ -9,7 +9,7 @@ use regex_lite::Regex;
use serde_json::json;
use std::collections::HashMap;
use std::{collections::HashMap, str::FromStr};
use windmill_parser::{Arg, MainArgSignature, Typ};
pub fn parse_bash_sig(code: &str) -> anyhow::Result<MainArgSignature> {
@@ -84,36 +84,94 @@ fn parse_bash_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
Ok(Some(args))
}
enum ParserState {
Normal,
InSingleQuote,
InDoubleQuote,
}
fn split_pwsh_args(code: &str) -> Vec<&str> {
let mut chars = code.char_indices().peekable();
let mut state = ParserState::Normal;
let mut splits = vec![];
let mut last_idx = 0;
while let Some((idx, char)) = chars.next() {
match (&state, char) {
(ParserState::Normal, '\'') => {
state = ParserState::InSingleQuote;
}
(ParserState::Normal, '"') => {
state = ParserState::InDoubleQuote;
}
(ParserState::InSingleQuote, '\'') => {
state = ParserState::Normal;
}
(ParserState::InDoubleQuote, '"') => {
state = ParserState::Normal;
}
(ParserState::Normal, ',') => {
splits.push(&code[last_idx..idx]);
last_idx = idx + 1; // skip the comma
}
_ => {}
}
}
if last_idx < code.len() {
splits.push(&code[last_idx..]);
}
splits
}
fn parse_powershell_file(code: &str) -> anyhow::Result<Option<Vec<Arg>>> {
let param_wrapper = RE_POWERSHELL_PARAM.captures(code);
let mut args = vec![];
if let Some(param_wrapper) = param_wrapper {
let param_wrapper = param_wrapper.get(1).unwrap().as_str();
for cap in RE_POWERSHELL_ARGS.captures_iter(param_wrapper) {
let typ = cap
.get(1)
.map(|x| x.as_str().to_string())
.unwrap_or("string".to_string());
let name = cap.get(2).unwrap().as_str().to_string();
let default = cap
.get(3)
.or(cap.get(4))
.map(|x| json!(x.as_str().to_string()));
let params = split_pwsh_args(param_wrapper);
for param in params {
if let Some(cap) = RE_POWERSHELL_ARGS.captures(param) {
let typ = cap.get(1).map(|x| x.as_str().to_string());
let name = cap.get(2).unwrap().as_str().to_string();
args.push(Arg {
name: name,
typ: match typ.as_str() {
"string" => Typ::Str(None),
"int" | "long" => Typ::Int,
"decimal" | "double" | "single" => Typ::Float,
"datetime" | "DateTime" => Typ::Datetime,
_ => Typ::Str(None),
},
default: default.clone(),
otyp: None,
has_default: default.is_some(),
oidx: None,
});
let mut parsed_typ = if let Some(typ) = typ {
match typ.as_str() {
"string" => Some(Typ::Str(None)),
"int" | "long" => Some(Typ::Int),
"decimal" | "double" | "single" => Some(Typ::Float),
"datetime" | "DateTime" => Some(Typ::Datetime),
_ => None,
}
} else {
None
};
let default = if let Some(x) = cap.get(3) {
Some(json!(x.as_str().to_string()))
} else if let Some(x) = cap.get(4) {
if parsed_typ.is_none() {
if x.as_str().parse::<i64>().is_ok() {
parsed_typ = Some(Typ::Int);
} else if x.as_str().parse::<f64>().is_ok() {
parsed_typ = Some(Typ::Float);
}
}
serde_json::Number::from_str(x.as_str())
.ok()
.map(serde_json::Value::Number)
} else {
None
};
args.push(Arg {
name: name,
typ: parsed_typ.unwrap_or(Typ::Str(None)),
default: default.clone(),
otyp: None,
has_default: default.is_some(),
oidx: None,
});
}
}
}
Ok(Some(args))
@@ -192,4 +250,77 @@ non_required="${5:-}"
Ok(())
}
#[test]
fn test_parse_powershell_sig() -> anyhow::Result<()> {
let code = r#"param($Msg, [string]$Msg2, $Dflt = "default value, with comma", [int]$Nb = 3 , $Nb2 = 5.0, $Nb3 = 5, $Wahoo = $env:WAHOO)"#;
assert_eq!(
parse_powershell_sig(code)?,
MainArgSignature {
star_args: false,
star_kwargs: false,
args: vec![
Arg {
otyp: None,
name: "Msg".to_string(),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
},
Arg {
otyp: None,
name: "Msg2".to_string(),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
},
Arg {
otyp: None,
name: "Dflt".to_string(),
typ: Typ::Str(None),
default: Some(json!("default value, with comma")),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Nb".to_string(),
typ: Typ::Int,
default: Some(json!(3)),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Nb2".to_string(),
typ: Typ::Float,
default: Some(json!(5.0)),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Nb3".to_string(),
typ: Typ::Int,
default: Some(json!(5)),
has_default: true,
oidx: None
},
Arg {
otyp: None,
name: "Wahoo".to_string(),
typ: Typ::Str(None),
default: None,
has_default: false,
oidx: None
}
],
no_main_func: None,
has_preprocessor: None
}
);
Ok(())
}
}
backend/src/main.rs
+90 -68
View File
@@ -8,7 +8,7 @@
use anyhow::Context;
use monitor::{
reload_delete_logs_periodically_setting, reload_indexer_config,
load_base_url, load_otel, reload_delete_logs_periodically_setting, reload_indexer_config,
reload_timeout_wait_result_setting, send_current_log_file_to_object_store,
send_logs_to_object_store,
};
@@ -37,7 +37,7 @@ use windmill_common::{
EXPOSE_METRICS_SETTING, EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, INDEXER_SETTING,
JOB_DEFAULT_TIMEOUT_SECS_SETTING, JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING,
LICENSE_KEY_SETTING, MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING,
OAUTH_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
OAUTH_SETTING, OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, SMTP_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
},
@@ -221,11 +221,66 @@ async fn windmill_main() -> anyhow::Result<()> {
let hostname = hostname();
#[cfg(not(feature = "flamegraph"))]
let _guard = windmill_common::tracing_init::initialize_tracing(&hostname);
let mut enable_standalone_indexer: bool = false;
let mode = std::env::var("MODE")
.map(|x| x.to_lowercase())
.map(|x| {
if &x == "server" {
println!("Binary is in 'server' mode");
Mode::Server
} else if &x == "worker" {
tracing::info!("Binary is in 'worker' mode");
#[cfg(windows)]
{
println!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL");
}
Mode::Worker
} else if &x == "agent" {
println!("Binary is in 'agent' mode");
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
}
#[cfg(not(feature = "enterprise"))]
{
panic!("Agent mode is only available in the EE, ignoring...");
}
#[cfg(feature = "enterprise")]
Mode::Agent
} else if &x == "indexer" {
tracing::info!("Binary is in 'indexer' mode");
#[cfg(not(feature = "tantivy"))]
{
eprintln!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features.");
panic!("Indexer mode requires compiling with the tantivy feature flag.");
}
#[cfg(feature = "tantivy")]
Mode::Indexer
} else if &x == "standalone+search"{
enable_standalone_indexer = true;
println!("Binary is in 'standalone' mode with search enabled");
Mode::Standalone
}
else {
if &x != "standalone" {
eprintln!("mode not recognized, defaulting to standalone: {x}");
} else {
println!("Binary is in 'standalone' mode");
}
Mode::Standalone
}
})
.unwrap_or_else(|_| {
tracing::info!("Mode not specified, defaulting to standalone");
Mode::Standalone
});
#[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))]
tracing::info!("jemalloc enabled");
println!("jemalloc enabled");
#[cfg(feature = "flamegraph")]
let _guard = windmill_common::tracing_init::setup_flamegraph();
@@ -236,13 +291,13 @@ async fn windmill_main() -> anyhow::Result<()> {
"cache" => {
#[cfg(feature = "embedding")]
{
tracing::info!("Caching embedding model...");
println!("Caching embedding model...");
windmill_api::embeddings::ModelInstance::load_model_files().await?;
tracing::info!("Cached embedding model");
println!("Cached embedding model");
}
#[cfg(not(feature = "embedding"))]
{
tracing::warn!("Embeddings are not enabled, ignoring...");
println!("Embeddings are not enabled, ignoring...");
}
cache_hub_scripts(std::env::args().nth(2)).await?;
@@ -256,64 +311,6 @@ async fn windmill_main() -> anyhow::Result<()> {
_ => {}
}
let mut enable_standalone_indexer: bool = false;
let mode = std::env::var("MODE")
.map(|x| x.to_lowercase())
.map(|x| {
if &x == "server" {
tracing::info!("Binary is in 'server' mode");
Mode::Server
} else if &x == "worker" {
tracing::info!("Binary is in 'worker' mode");
#[cfg(windows)]
{
tracing::warn!("It is highly recommended to use the agent mode instead on windows (MODE=agent) and to pass a BASE_INTERNAL_URL");
}
Mode::Worker
} else if &x == "agent" {
tracing::info!("Binary is in 'agent' mode");
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
tracing::warn!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
}
#[cfg(not(feature = "enterprise"))]
{
panic!("Agent mode is only available in the EE, ignoring...");
}
#[cfg(feature = "enterprise")]
Mode::Agent
} else if &x == "indexer" {
tracing::info!("Binary is in 'indexer' mode");
#[cfg(not(feature = "tantivy"))]
{
tracing::error!("Cannot start the indexer because tantivy is not included in this binary/image. Make sure you are using the EE image if you want to access the full text search features.");
panic!("Indexer mode requires compiling with the tantivy feature flag.");
}
#[cfg(feature = "tantivy")]
Mode::Indexer
} else if &x == "standalone+search"{
enable_standalone_indexer = true;
tracing::info!("Binary is in 'standalone' mode with search enabled");
Mode::Standalone
}
else {
if &x != "standalone" {
tracing::error!("mode not recognized, defaulting to standalone: {x}");
} else {
tracing::info!("Binary is in 'standalone' mode");
}
Mode::Standalone
}
})
.unwrap_or_else(|_| {
tracing::info!("Mode not specified, defaulting to standalone");
Mode::Standalone
});
#[allow(unused_mut)]
let mut num_workers = if mode == Mode::Server || mode == Mode::Indexer {
0
@@ -325,7 +322,7 @@ async fn windmill_main() -> anyhow::Result<()> {
};
if num_workers > 1 {
tracing::warn!(
println!(
"We STRONGLY recommend using at most 1 worker per container, use at your own risks"
);
}
@@ -347,10 +344,26 @@ async fn windmill_main() -> anyhow::Result<()> {
IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1))
};
tracing::info!("Connecting to database...");
println!("Connecting to database...");
let db = windmill_common::connect_db(server_mode, indexer_mode).await?;
load_otel(&db).await;
tracing::info!("Database connected");
let environment = load_base_url(&db)
.await
.unwrap_or_else(|_| "local".to_string())
.trim_start_matches("https://")
.trim_start_matches("http://")
.split(".")
.next()
.unwrap_or_else(|| "local")
.to_string();
#[cfg(not(feature = "flamegraph"))]
let _guard = windmill_common::tracing_init::initialize_tracing(&hostname, &mode, &environment);
let num_version = sqlx::query_scalar!("SELECT version()").fetch_one(&db).await;
tracing::info!(
@@ -776,6 +789,15 @@ Windmill Community Edition {GIT_VERSION}
tracing::error!(error = %e, "Could not reload debug metrics setting");
}
},
OTEL_SETTING => {
tracing::info!("OTEL setting changed, restarting");
// we wait a bit randomly to avoid having all servers and workers shutdown at same time
let rd_delay = rand::thread_rng().gen_range(0..4);
tokio::time::sleep(Duration::from_secs(rd_delay)).await;
if let Err(e) = tx.send(()) {
tracing::error!(error = %e, "Could not send killpill");
}
},
REQUEST_SIZE_LIMIT_SETTING => {
if server_mode {
tracing::info!("Request limit size change detected, killing server expecting to be restarted");
backend/src/monitor.rs
+73 -9
View File
@@ -12,7 +12,7 @@ use std::{
use chrono::{NaiveDateTime, Utc};
use futures::{stream::FuturesUnordered, StreamExt};
use serde::de::DeserializeOwned;
use serde::{de::DeserializeOwned, Deserializer};
use sqlx::{Pool, Postgres};
use tokio::{
join,
@@ -40,7 +40,7 @@ use windmill_common::{
EXTRA_PIP_INDEX_URL_SETTING, HUB_BASE_URL_SETTING, JOB_DEFAULT_TIMEOUT_SECS_SETTING,
JWT_SECRET_SETTING, KEEP_JOB_DIR_SETTING, LICENSE_KEY_SETTING,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING, NPM_CONFIG_REGISTRY_SETTING, OAUTH_SETTING,
PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
OTEL_SETTING, PIP_INDEX_URL_SETTING, REQUEST_SIZE_LIMIT_SETTING,
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
},
@@ -58,7 +58,8 @@ use windmill_common::{
},
BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL,
HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED,
MONITOR_LOGS_ON_OBJECT_STORE, SERVICE_LOG_RETENTION_SECS,
MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED,
SERVICE_LOG_RETENTION_SECS,
};
use windmill_queue::cancel_job;
use windmill_worker::{
@@ -199,6 +200,65 @@ pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> {
Ok(())
}
fn empty_string_as_none<'de, D>(deserializer: D) -> Result<Option<String>, D::Error>
where
D: Deserializer<'de>,
{
let option = <Option<String> as serde::Deserialize>::deserialize(deserializer)?;
Ok(option.filter(|s| !s.is_empty()))
}
#[derive(serde::Deserialize)]
struct OtelSetting {
metrics_enabled: Option<bool>,
logs_enabled: Option<bool>,
tracing_enabled: Option<bool>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_endpoint: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_headers: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_protocol: Option<String>,
#[serde(default, deserialize_with = "empty_string_as_none")]
otel_exporter_otlp_compression: Option<String>,
}
pub async fn load_otel(db: &DB) {
let otel = load_value_from_global_settings(db, OTEL_SETTING).await;
if let Ok(v) = otel {
if let Some(v) = v {
let deser = serde_json::from_value::<OtelSetting>(v);
if let Ok(o) = deser {
let metrics_enabled = o.metrics_enabled.unwrap_or(false);
let logs_enabled = o.logs_enabled.unwrap_or(false);
let tracing_enabled = o.tracing_enabled.unwrap_or(false);
OTEL_METRICS_ENABLED.store(metrics_enabled, Ordering::Relaxed);
OTEL_LOGS_ENABLED.store(logs_enabled, Ordering::Relaxed);
OTEL_TRACING_ENABLED.store(tracing_enabled, Ordering::Relaxed);
if let Some(endpoint) = o.otel_exporter_otlp_endpoint.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_ENDPOINT", endpoint);
}
if let Some(headers) = o.otel_exporter_otlp_headers.as_ref() {
std::env::set_var("OTEL_EXPORTER_OTLP_HEADERS", headers);
}
if let Some(protocol) = o.otel_exporter_otlp_protocol {
std::env::set_var("OTEL_EXPORTER_OTLP_PROTOCOL", protocol);
}
if let Some(compression) = o.otel_exporter_otlp_compression {
std::env::set_var("OTEL_EXPORTER_OTLP_COMPRESSION", compression);
}
println!("OTEL settings loaded: tracing ({tracing_enabled}), logs ({logs_enabled}), metrics ({metrics_enabled}), endpoint ({:?}), headers defined: ({})",
o.otel_exporter_otlp_endpoint, o.otel_exporter_otlp_headers.is_some());
} else {
tracing::error!("Error deserializing otel settings");
}
}
} else {
tracing::error!("Error loading otel settings: {}", otel.unwrap_err());
}
}
pub async fn load_tag_per_workspace_enabled(db: &DB) -> error::Result<()> {
let metrics_enabled =
load_value_from_global_settings(db, DEFAULT_TAGS_PER_WORKSPACE_SETTING).await;
@@ -1337,7 +1397,7 @@ pub async fn reload_worker_config(
}
}
pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
pub async fn load_base_url(db: &DB) -> error::Result<String> {
let q_base_url = load_value_from_global_settings(db, BASE_URL_SETTING).await?;
let std_base_url = std::env::var("BASE_URL")
@@ -1361,6 +1421,14 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
std_base_url
};
{
let mut l = BASE_URL.write().await;
*l = base_url.clone();
}
Ok(base_url)
}
pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?;
let oauths = if let Some(q) = q_oauth {
@@ -1374,6 +1442,7 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
None
};
let base_url = load_base_url(db).await?;
let is_secure = base_url.starts_with("https://");
{
@@ -1383,11 +1452,6 @@ pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
.unwrap();
}
{
let mut l = BASE_URL.write().await;
*l = base_url
}
{
let mut l = IS_SECURE.write().await;
*l = is_secure;
backend/tests/worker.rs
+5 -1
View File
@@ -77,7 +77,11 @@ async fn initialize_tracing() {
static ONCE: Once = Once::new();
ONCE.call_once(|| {
let _ = windmill_common::tracing_init::initialize_tracing("test");
let _ = windmill_common::tracing_init::initialize_tracing(
"test",
&windmill_common::utils::Mode::Standalone,
"test",
);
});
}
backend/windmill-api/openapi-deref.yaml
+6
View File
@@ -8709,6 +8709,9 @@ paths:
- identity
- deploymentcallback
- singlescriptflow
- flowscript
- flownode
- appscript
schedule_path:
type: string
permissioned_as:
@@ -9267,6 +9270,9 @@ paths:
- identity
- deploymentcallback
- singlescriptflow
- flowscript
- flownode
- appscript
schedule_path:
type: string
permissioned_as:
backend/windmill-api/openapi.yaml
+28 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.435.1
version: 1.437.1
title: Windmill API
contact:
@@ -5467,6 +5467,23 @@ paths:
schema:
$ref: "#/components/schemas/AppWithLastVersion"
/w/{workspace}/apps/get/lite/{path}:
get:
summary: get app lite by path
operationId: getAppLiteByPath
tags:
- app
parameters:
- $ref: "#/components/parameters/WorkspaceId"
- $ref: "#/components/parameters/ScriptPath"
responses:
"200":
description: app lite details
content:
application/json:
schema:
$ref: "#/components/schemas/AppWithLastVersion"
/w/{workspace}/apps/get/draft/{path}:
get:
summary: get app by path with draft
@@ -5791,6 +5808,8 @@ paths:
#flow: flow/<path>
path:
type: string
version:
type: integer
args: {}
raw_code:
type: object
@@ -5808,6 +5827,8 @@ paths:
required:
- content
- language
id:
type: integer
force_viewer_static_fields:
type: object
force_viewer_one_of_fields:
@@ -10964,6 +10985,8 @@ components:
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
]
schedule_path:
type: string
@@ -11085,6 +11108,8 @@ components:
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
]
schedule_path:
type: string
@@ -11195,6 +11220,8 @@ components:
type: string
is_admin:
type: boolean
name:
type: string
is_super_admin:
type: boolean
created_at:
backend/windmill-api/src/apps.rs
+200 -160
View File
@@ -1,4 +1,4 @@
use std::collections::HashMap;
use std::{collections::HashMap, sync::Arc};
/*
* Author: Ruben Fiszel
@@ -31,6 +31,7 @@ use axum::{
routing::{delete, get, post},
Router,
};
use futures::future::{FutureExt, TryFutureExt};
use hyper::StatusCode;
#[cfg(feature = "parquet")]
use itertools::Itertools;
@@ -50,13 +51,15 @@ use windmill_audit::ActionKind;
#[cfg(feature = "parquet")]
use windmill_common::s3_helpers::build_object_store_client;
use windmill_common::{
apps::ListAppQuery,
apps::{AppScriptId, ListAppQuery},
cache::{self, future::FutureCachedExt},
db::UserDB,
error::{to_anyhow, Error, JsonResult, Result},
jobs::{get_payload_tag_from_prefixed_path, JobPayload, RawCode},
users::username_to_permissioned_as,
utils::{
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath
http_get_from_hub, not_found_if_none, paginate, query_elems_from_hub, require_admin,
Pagination, StripPath,
},
variables::{build_crypt, build_crypt_with_key_suffix},
worker::{to_raw_value, CLOUD_HOSTED},
@@ -71,6 +74,7 @@ pub fn workspaced_service() -> Router {
.route("/list", get(list_apps))
.route("/list_search", get(list_search_apps))
.route("/get/p/*path", get(get_app))
.route("/get/lite/*path", get(get_app_lite))
.route("/get/draft/*path", get(get_app_w_draft))
.route("/secret_of/*path", get(get_secret_id))
.route("/get/v/*id", get(get_app_by_id))
@@ -191,15 +195,16 @@ pub type StaticFields = HashMap<String, Box<RawValue>>;
pub type OneOfFields = HashMap<String, Vec<Box<RawValue>>>;
pub type AllowUserResources = Vec<String>;
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone, Default)]
#[serde(rename_all = "lowercase")]
pub enum ExecutionMode {
#[default]
Anonymous,
Publisher,
Viewer,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct PolicyTriggerableInputs {
static_inputs: StaticFields,
one_of_inputs: OneOfFields,
@@ -215,7 +220,7 @@ pub struct S3Input {
file_key_regex: String,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
pub struct Policy {
pub on_behalf_of: Option<String>,
pub on_behalf_of_email: Option<String>,
@@ -410,6 +415,33 @@ async fn get_app(
Ok(Json(app))
}
async fn get_app_lite(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<AppWithLastVersion> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
app.extra_perms, coalesce(app_version_lite.value::json, app_version.value) as value,
app_version.created_at, app_version.created_by, NULL as starred
FROM app, app_version
LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
WHERE app.path = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]",
)
.bind(path.to_owned())
.bind(&w_id)
.fetch_optional(&mut *tx)
.await?;
tx.commit().await?;
let app = not_found_if_none(app_o, "App", path)?;
Ok(Json(app))
}
async fn get_app_w_draft(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
@@ -527,12 +559,11 @@ async fn update_app_history(
return Ok(());
}
async fn custom_path_exists(
Extension(db): Extension<DB>,
Path((w_id, custom_path)): Path<(String, String)>,
) -> JsonResult<bool> {
let exists =
let exists =
sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2))",
custom_path,
@@ -583,8 +614,9 @@ async fn get_public_app_by_secret(
let app_o = sqlx::query_as::<_, AppWithLastVersion>(
"SELECT app.id, app.path, app.summary, app.versions, app.policy,
null as extra_perms, app_version.value,
null as extra_perms, coalesce(app_version_lite.value::json, app_version.value::json) as value,
app_version.created_at, app_version.created_by from app, app_version
LEFT JOIN app_version_lite ON app_version_lite.id = app_version.id
WHERE app.id = $1 AND app.workspace_id = $2 AND app_version.id = app.versions[array_upper(app.versions, 1)]")
.bind(&id)
.bind(&w_id)
@@ -626,7 +658,6 @@ async fn get_public_app_by_secret(
Ok(Json(app))
}
async fn get_public_resource(
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
@@ -710,7 +741,6 @@ async fn create_app(
}
if let Some(custom_path) = &app.custom_path {
require_admin(authed.is_admin, &authed.username)?;
let exists = sqlx::query_scalar!(
@@ -746,7 +776,9 @@ async fn create_app(
app.summary,
json!(app.policy),
app.draft_only,
app.custom_path,
app.custom_path
.map(|s| if s.is_empty() { None } else { Some(s) })
.flatten()
)
.fetch_one(&mut *tx)
.await?;
@@ -987,26 +1019,29 @@ async fn update_app(
}
if let Some(ncustom_path) = &ns.custom_path {
require_admin(authed.is_admin, &authed.username)?;
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))",
ncustom_path,
if *CLOUD_HOSTED { Some(&w_id) } else { None },
path,
w_id
)
.fetch_one(&mut *tx)
.await?.unwrap_or(false);
if ncustom_path.is_empty() {
sqlb.set("custom_path", "NULL");
} else {
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM app WHERE custom_path = $1 AND ($2::TEXT IS NULL OR workspace_id = $2) AND NOT (path = $3 AND workspace_id = $4))",
ncustom_path,
if *CLOUD_HOSTED { Some(&w_id) } else { None },
path,
w_id
)
.fetch_one(&mut *tx)
.await?.unwrap_or(false);
if exists {
return Err(Error::BadRequest(format!(
"App with custom path {} already exists",
ncustom_path
)));
if exists {
return Err(Error::BadRequest(format!(
"App with custom path {} already exists",
ncustom_path
)));
}
sqlb.set_str("custom_path", ncustom_path);
}
sqlb.set_str("custom_path", ncustom_path);
}
if let Some(mut npolicy) = ns.policy {
@@ -1144,6 +1179,10 @@ async fn update_app(
#[derive(Debug, Deserialize, Clone)]
pub struct ExecuteApp {
/// The app version to execute. Fallback to `path` if not provided.
pub version: Option<i64>,
/// The app script id (from the `app_script` table) to execute.
pub id: Option<i64>,
pub args: HashMap<String, Box<RawValue>>,
// - script: script/<path>
// - flow: flow/<path>
@@ -1206,6 +1245,22 @@ async fn get_on_behalf_details_from_policy_and_authed(
Ok((username, permissioned_as, email))
}
/// Convert the triggerables from the old format to the new format.
fn empty_triggerables(mut policy: Policy) -> Policy {
use std::mem::take;
if let Some(triggerables) = take(&mut policy.triggerables) {
let mut triggerables_v2 = take(&mut policy.triggerables_v2).unwrap_or_default();
for (k, static_inputs) in triggerables.into_iter() {
triggerables_v2.insert(
k,
PolicyTriggerableInputs { static_inputs, ..Default::default() },
);
}
policy.triggerables_v2 = Some(triggerables_v2);
}
policy
}
async fn execute_component(
OptAuthed(opt_authed): OptAuthed,
Extension(db): Extension<DB>,
@@ -1228,99 +1283,129 @@ async fn execute_component(
};
let path = path.to_path();
let (arc_policy, policy): (Arc<Policy>, Policy);
let policy_triggerables_default = Default::default();
let policy = match payload.clone() {
// Two cases here:
// 1. The component is executed from the editor (i.e. in "preview" mode), then:
// - The policy is set to default (in `Viewer` execution mode).
// - The policy triggerables are built by the frontend and retrieved from the request
// payload.
// - In case of inline script, the `RawCode` from the request is pushed as is to the
// job queue.
// 2. Otherwise (i.e. "run" mode):
// - The policy and triggerables are fetched from the database.
// - In case of inline script, if an entry exists in the `app_script` table, push
// an `AppScript` job payload, as in (.1) otherwise.
let (policy, policy_triggerables) = match payload {
// 1. "preview" mode.
ExecuteApp {
force_viewer_static_fields: Some(static_fields),
force_viewer_one_of_fields: Some(one_of_fields),
force_viewer_static_fields: Some(static_inputs),
force_viewer_one_of_fields: Some(one_of_inputs),
force_viewer_allow_user_resources: Some(allow_user_resources),
..
} => {
let mut hm = HashMap::new();
if let Some(path) = payload.path.clone() {
hm.insert(
format!("{}:{path}", payload.component),
PolicyTriggerableInputs {
static_inputs: static_fields,
one_of_inputs: one_of_fields,
allow_user_resources,
},
);
} else {
hm.insert(
format!(
"{}:{}",
payload.component,
digest(payload.raw_code.clone().unwrap().content.as_str())
),
PolicyTriggerableInputs {
static_inputs: static_fields,
one_of_inputs: one_of_fields,
allow_user_resources,
},
);
}
Policy {
execution_mode: ExecutionMode::Viewer,
triggerables: None,
triggerables_v2: Some(hm),
on_behalf_of: None,
on_behalf_of_email: None,
s3_inputs: None,
}
}
} => (
&Policy { execution_mode: ExecutionMode::Viewer, ..Default::default() },
&PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources },
),
// 2. "run" mode.
_ => {
let policy_o = sqlx::query_scalar!(
"SELECT policy from app WHERE path = $1 AND workspace_id = $2",
// Policy is fetched from the database on app `path` and `workspace_id`.
let policy_fut = sqlx::query_scalar!(
"SELECT policy as \"policy: sqlx::types::Json<Box<RawValue>>\"
FROM app WHERE app.path = $1 AND app.workspace_id = $2 LIMIT 1",
path,
&w_id
&w_id,
)
.fetch_optional(&db)
.await?;
.map_err(Into::<Error>::into)
.map(|policy_o| Result::Ok(not_found_if_none(policy_o?, "App", path)?))
.map(|policy| Result::Ok(serde_json::from_str(policy?.get())?))
.map_ok(empty_triggerables);
let policy = not_found_if_none(policy_o, "App", path)?;
// 1. The app `version` is provided: cache the fetched policy.
// 2. Otherwise, always fetch the policy from the database.
let policy = if let Some(id) = payload.version {
let cache = cache::anon!({ u64 => Arc<Policy> } in "policy" <= 1000);
arc_policy = policy_fut
.map_ok(Arc::new)
.cached(cache, &(id as u64))
.await?;
&*arc_policy
} else {
policy = policy_fut.await?;
&policy
};
serde_json::from_value::<Policy>(policy).map_err(to_anyhow)?
// Compute the path for the triggerables map:
// - flow: `flow/<payload.path>`
// - script: `script/<payload.path>`
// - inline script: `rawscript/<sha256(raw_code.content)>`
let path = match &payload {
// flow or script: just use the `payload.path`.
ExecuteApp { path: Some(path), .. } => path,
// inline script: without entry in the `app_script` table.
ExecuteApp { raw_code: Some(raw_code), id: None, .. } => &digest(&raw_code.content),
// inline script: with an entry in the `app_script` table.
ExecuteApp { raw_code: Some(_), id: Some(id), .. } => {
let cache = cache::anon!({ u64 => Arc<String> } in "appscriptpath" <= 10000);
// `id` is unique, cache the result.
&*sqlx::query_scalar!(
"SELECT format('rawscript/%s', code_sha256) as \"path!: String\"
FROM app_script WHERE id = $1 LIMIT 1",
id
)
.fetch_one(&db)
.map_err(Into::<Error>::into)
.map_ok(Arc::new)
.cached(cache, &(*id as u64))
.await?
}
_ => unreachable!(),
};
// Retrieve the triggerables from the policy on `path` or `<component>:<path>`.
let triggerables_v2 = policy
.triggerables_v2
.as_ref()
.ok_or_else(|| Error::BadRequest(format!("Policy is missing triggerables")))?;
let policy_triggerables = triggerables_v2
.get(path) // start with `path` in case we can avoid the next` format!`.
.or_else(|| triggerables_v2.get(&format!("{}:{}", payload.component, &path)))
.or(match policy.execution_mode {
ExecutionMode::Viewer => Some(&policy_triggerables_default),
_ => None,
})
.ok_or_else(|| Error::BadRequest(format!("Path {path} forbidden by policy")))?;
(policy, policy_triggerables)
}
};
let (username, permissioned_as, email) =
get_on_behalf_details_from_policy_and_authed(&policy, &opt_authed).await?;
let (job_payload, (args, job_id), tag) = match payload {
ExecuteApp { args, component, raw_code: Some(raw_code), path: None, .. } => {
let content = &raw_code.content;
let payload = JobPayload::Code(raw_code.clone());
let path = digest(content);
let args = build_args(
policy,
&component,
path,
args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
(payload, args, None)
}
ExecuteApp { args, component, raw_code: None, path: Some(path), .. } => {
let (payload, tag) = get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?;
let args = build_args(
policy,
&component,
path.to_string(),
args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
(payload, args, tag)
}
let (args, job_id) = build_args(
policy,
policy_triggerables,
payload.args,
opt_authed.as_ref(),
&user_db,
&db,
&w_id,
)
.await?;
let (job_payload, tag) = match (payload.path, payload.raw_code, payload.id) {
// flow or script:
(Some(path), None, None) => get_payload_tag_from_prefixed_path(&path, &db, &w_id).await?,
// inline script: in "preview" mode or without entry in the `app_script` table.
(None, Some(raw_code), None) => (JobPayload::Code(raw_code), None),
// inline script: in "run" mode and with an entry in the `app_script` table.
(None, Some(RawCode { language, path, cache_ttl, .. }), Some(id)) => (
JobPayload::AppScript { id: AppScriptId(id), cache_ttl, language, path },
None,
),
_ => unreachable!(),
};
let tx = windmill_queue::PushIsolationLevel::IsolatedRoot(db.clone());
@@ -1655,9 +1740,12 @@ async fn exists_app(
}
async fn build_args(
policy: Policy,
component: &str,
path: String,
policy: &Policy,
PolicyTriggerableInputs {
static_inputs,
one_of_inputs,
allow_user_resources,
}: &PolicyTriggerableInputs,
mut args: HashMap<String, Box<RawValue>>,
authed: Option<&ApiAuthed>,
user_db: &UserDB,
@@ -1665,54 +1753,6 @@ async fn build_args(
w_id: &str,
) -> Result<(PushArgsOwned, Option<Uuid>)> {
let mut job_id: Option<Uuid> = None;
let key = format!("{}:{}", component, &path);
let (static_inputs, one_of_inputs, allow_user_resources) = match policy {
Policy { triggerables_v2: Some(t), .. } => {
let PolicyTriggerableInputs { static_inputs, one_of_inputs, allow_user_resources } = t
.get(&key)
.or_else(|| t.get(&path))
.map(|x| x.clone())
.or_else(|| {
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
Some(PolicyTriggerableInputs {
static_inputs: HashMap::new(),
one_of_inputs: HashMap::new(),
allow_user_resources: Vec::new(),
})
} else {
None
}
})
.ok_or_else(|| {
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
})?;
(static_inputs, one_of_inputs, allow_user_resources)
}
Policy { triggerables: Some(t), .. } => {
let static_inputs = t
.get(&key)
.or_else(|| t.get(&path))
.map(|x| x.clone())
.or_else(|| {
if matches!(policy.execution_mode, ExecutionMode::Viewer) {
Some(HashMap::new())
} else {
None
}
})
.ok_or_else(|| {
Error::BadRequest(format!("path {} is not allowed in the app policy", path))
})?;
(static_inputs, HashMap::new(), Vec::new())
}
_ => Err(Error::BadRequest(format!(
"Policy is missing triggerables for {}",
key
)))?,
};
let mut safe_args = HashMap::<String, Box<RawValue>>::new();
// tracing::error!("{:?}", allow_user_resources);
@@ -1761,16 +1801,16 @@ async fn build_args(
}
for (k, v) in one_of_inputs {
if safe_args.contains_key(&k) {
if safe_args.contains_key(k) {
continue;
}
if let Some(arg_val) = args.get(&k) {
if let Some(arg_val) = args.get(k) {
let arg_str = arg_val.get();
let options_str_vec = v.iter().map(|x| x.get()).collect::<Vec<&str>>();
if options_str_vec.contains(&arg_str) {
safe_args.insert(k.to_string(), arg_val.clone());
args.remove(&k);
args.remove(k);
continue;
}
@@ -1781,7 +1821,7 @@ async fn build_args(
.all(|x| options_str_vec.contains(&x.get()))
{
safe_args.insert(k.to_string(), arg_val.clone());
args.remove(&k);
args.remove(k);
continue;
}
}
backend/windmill-api/src/args.rs
+469
View File
@@ -0,0 +1,469 @@
use std::collections::HashMap;
#[cfg(feature = "parquet")]
use crate::job_helpers_ee::get_workspace_s3_resource;
use axum::{
extract::{FromRequest, FromRequestParts, Query, Request},
http::{HeaderMap, Uri},
response::{IntoResponse, Response},
};
use bytes::Bytes;
use http::{header::CONTENT_TYPE, request::Parts};
#[cfg(feature = "parquet")]
use object_store::{Attribute, Attributes};
use serde::Deserialize;
use serde_json::value::RawValue;
use sqlx::types::JsonRawValue;
#[cfg(feature = "parquet")]
use windmill_common::s3_helpers::build_object_store_client;
use windmill_common::{error::Error, worker::to_raw_value, DB};
use windmill_queue::PushArgsOwned;
use crate::db::ApiAuthed;
#[cfg(feature = "parquet")]
use crate::job_helpers_ee::{get_random_file_name, upload_file_internal};
pub struct WebhookArgs {
pub args: PushArgsOwned,
pub file_req: Option<Request>,
}
impl WebhookArgs {
#[cfg(not(feature = "parquet"))]
pub async fn to_push_args_owned(
self,
_authed: &ApiAuthed,
_db: &DB,
_w_id: &str,
) -> Result<PushArgsOwned, Error> {
return Err(Error::BadRequest(format!(
"Uploading files requires the parquet feature"
)));
}
#[cfg(feature = "parquet")]
pub async fn to_push_args_owned(
mut self,
authed: &ApiAuthed,
db: &DB,
w_id: &str,
) -> Result<PushArgsOwned, Error> {
if let Some(req) = self.file_req {
{
let (_, s3_resource) =
get_workspace_s3_resource(authed, db, None, "", w_id, None).await?;
if let Some(s3_resource) = s3_resource {
let s3_client = build_object_store_client(&s3_resource).await?;
let content_type = req
.headers()
.get(CONTENT_TYPE)
.map(|x| x.to_str().ok().map(|x| x.to_string()))
.flatten();
let file_extension = content_type
.as_ref()
.map(|mime_str| {
mime_guess::get_mime_extensions_str(mime_str)
.map(|x| x.first().map(|x| x.to_string()))
})
.flatten()
.flatten();
let file_key = get_random_file_name(file_extension);
let options = Attributes::from_iter(vec![
(
Attribute::ContentType,
content_type.unwrap_or("application/octet-stream".to_string()),
),
(Attribute::ContentDisposition, "inline".to_string()),
])
.into();
upload_file_internal(s3_client, &file_key, req, options).await?;
self.args.args.insert(
"body".to_string(),
to_raw_value(&serde_json::json!({
"s3": &file_key
})),
);
return Ok(self.args);
}
}
return Err(Error::BadRequest(format!(
"You need to connect your workspace to an S3 bucket to upload files"
)));
}
Ok(self.args)
}
}
#[derive(Deserialize)]
pub struct RequestQuery {
pub raw: Option<bool>,
pub wrap_body: Option<bool>,
pub include_header: Option<String>,
}
#[axum::async_trait]
impl<S> FromRequest<S, axum::body::Body> for WebhookArgs
where
S: Send + Sync,
{
type Rejection = Response;
async fn from_request(
req: Request<axum::body::Body>,
_state: &S,
) -> Result<Self, Self::Rejection> {
let (content_type, mut extra, use_raw, wrap_body) = {
let headers_map = req.headers();
let content_type_header = headers_map.get(CONTENT_TYPE);
let content_type = content_type_header.and_then(|value| value.to_str().ok());
let uri = req.uri();
let query = Query::<RequestQuery>::try_from_uri(uri).unwrap().0;
let mut extra = build_extra(&headers_map, query.include_header);
let query_decode = DecodeQueries::from_uri(uri);
if let Some(DecodeQueries(queries)) = query_decode {
extra.extend(queries);
}
let raw = query.raw.as_ref().is_some_and(|x| *x);
let wrap_body = query.wrap_body.as_ref().is_some_and(|x| *x);
(content_type, extra, raw, wrap_body)
};
let no_content_type = content_type.is_none();
if no_content_type || content_type.unwrap().starts_with("application/json") {
let bytes = Bytes::from_request(req, _state)
.await
.map_err(IntoResponse::into_response)?;
if no_content_type && bytes.is_empty() {
if use_raw {
extra.insert("raw_string".to_string(), to_raw_value(&"".to_string()));
}
let mut args = HashMap::new();
if wrap_body {
args.insert("body".to_string(), to_raw_value(&serde_json::json!({})));
}
return Ok(Self {
args: PushArgsOwned { extra: Some(extra), args: args },
file_req: None,
});
}
let str = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
PushArgsOwned::from_json(extra, use_raw, wrap_body, str)
.await
.map(|args| Self { args, file_req: None })
} else if content_type
.unwrap()
.starts_with("application/cloudevents+json")
{
let bytes = Bytes::from_request(req, _state)
.await
.map_err(IntoResponse::into_response)?;
let str = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
PushArgsOwned::from_ce_json(extra, use_raw, str)
.await
.map(|args| Self { args, file_req: None })
} else if content_type
.unwrap()
.starts_with("application/cloudevents-batch+json")
{
Err(
Error::BadRequest(format!("Cloud events batching is not supported yet"))
.into_response(),
)
} else if content_type.unwrap().starts_with("text/plain") {
let bytes = Bytes::from_request(req, _state)
.await
.map_err(IntoResponse::into_response)?;
let str = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
extra.insert("raw_string".to_string(), to_raw_value(&str));
Ok(Self {
args: PushArgsOwned { extra: Some(extra), args: HashMap::new() },
file_req: None,
})
} else if content_type
.unwrap()
.starts_with("application/x-www-form-urlencoded")
{
let bytes = Bytes::from_request(req, _state)
.await
.map_err(IntoResponse::into_response)?;
if use_raw {
let raw_string = String::from_utf8(bytes.to_vec()).map_err(|e| {
Error::BadRequest(format!("invalid utf8: {}", e)).into_response()
})?;
extra.insert("raw_string".to_string(), to_raw_value(&raw_string));
}
let payload: HashMap<String, Option<String>> = serde_urlencoded::from_bytes(&bytes)
.map_err(|e| {
Error::BadRequest(format!("invalid urlencoded data: {}", e)).into_response()
})?;
let payload = payload
.into_iter()
.map(|(k, v)| (k, to_raw_value(&v)))
.collect::<HashMap<_, _>>();
return Ok(Self {
args: PushArgsOwned { extra: Some(extra), args: payload },
file_req: None,
});
} else {
return Ok(Self {
args: PushArgsOwned { extra: None, args: HashMap::new() },
file_req: Some(req),
});
// Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response())
}
}
}
lazy_static::lazy_static! {
static ref INCLUDE_HEADERS: Vec<String> = std::env::var("INCLUDE_HEADERS")
.ok().map(|x| x
.split(',')
.map(|s| s.to_string())
.collect()).unwrap_or_default();
}
pub fn build_extra(
headers: &HeaderMap,
include_header: Option<String>,
) -> HashMap<String, Box<RawValue>> {
let mut args = HashMap::new();
let whitelist = include_header
.map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
.unwrap_or_default();
whitelist
.iter()
.chain(INCLUDE_HEADERS.iter())
.for_each(|h| {
if let Some(v) = headers.get(h) {
args.insert(
h.to_string().to_lowercase().replace('-', "_"),
to_raw_value(&v.to_str().unwrap().to_string()),
);
}
});
args
}
#[derive(Deserialize)]
pub struct IncludeQuery {
pub include_query: Option<String>,
}
pub struct DecodeQueries(pub HashMap<String, Box<RawValue>>);
#[axum::async_trait]
impl<S> FromRequestParts<S> for DecodeQueries
where
S: Send + Sync,
{
type Rejection = Response;
async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result<Self, Self::Rejection> {
Ok(DecodeQueries::from_uri(&parts.uri).unwrap_or_else(|| DecodeQueries(HashMap::new())))
}
}
impl DecodeQueries {
pub fn from_uri(uri: &Uri) -> Option<Self> {
let query = uri.query();
if query.is_none() {
return None;
}
let query = query.unwrap();
let include_query = serde_urlencoded::from_str::<IncludeQuery>(query)
.map(|x| x.include_query)
.ok()
.flatten()
.unwrap_or_default();
let parse_query_args = include_query
.split(",")
.map(|s| s.to_string())
.collect::<Vec<_>>();
let mut args = HashMap::new();
if !parse_query_args.is_empty() {
let queries =
serde_urlencoded::from_str::<HashMap<String, String>>(query).unwrap_or_default();
parse_query_args.iter().for_each(|h| {
if let Some(v) = queries.get(h) {
args.insert(h.to_string(), to_raw_value(v));
}
});
}
Some(DecodeQueries(args))
}
}
// impl<'c> PushArgs<'c> {
// pub fn insert<K: Into<String>, V: Into<Box<RawValue>>>(&mut self, k: K, v: V) {
// self.extra.insert(k.into(), v.into());
// }
// }
fn restructure_cloudevents_metadata(
mut p: HashMap<String, Box<RawValue>>,
) -> Result<HashMap<String, Box<RawValue>>, Error> {
let data = p
.remove("data")
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
let str = data.to_string();
let wrap_body = str.len() > 0 && str.chars().next().unwrap() != '{';
if wrap_body {
let args = serde_json::from_str::<Option<Box<RawValue>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
let mut hm = HashMap::new();
hm.insert("body".to_string(), args);
hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p));
Ok(hm)
} else {
let mut hm = serde_json::from_str::<Option<HashMap<String, Box<JsonRawValue>>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?
.unwrap_or_else(HashMap::new);
hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p));
Ok(hm)
}
}
trait PushArgsOwnedExt: Sized {
async fn from_json(
extra: HashMap<String, Box<RawValue>>,
use_raw: bool,
force_wrap_body: bool,
str: String,
) -> Result<Self, Response>;
async fn from_ce_json(
extra: HashMap<String, Box<RawValue>>,
use_raw: bool,
str: String,
) -> Result<Self, Response>;
}
impl PushArgsOwnedExt for PushArgsOwned {
async fn from_json(
mut extra: HashMap<String, Box<RawValue>>,
use_raw: bool,
force_wrap_body: bool,
str: String,
) -> Result<Self, Response> {
if use_raw {
extra.insert("raw_string".to_string(), to_raw_value(&str));
}
let wrap_body = force_wrap_body || str.len() > 0 && str.chars().next().unwrap() != '{';
if wrap_body {
let args = serde_json::from_str::<Option<Box<RawValue>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())?
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
let mut hm = HashMap::new();
hm.insert("body".to_string(), args);
Ok(PushArgsOwned { extra: Some(extra), args: hm })
} else {
let hm = serde_json::from_str::<Option<HashMap<String, Box<JsonRawValue>>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())?
.unwrap_or_else(HashMap::new);
Ok(PushArgsOwned { extra: Some(extra), args: hm })
}
}
async fn from_ce_json(
mut extra: HashMap<String, Box<RawValue>>,
use_raw: bool,
str: String,
) -> Result<Self, Response> {
if use_raw {
extra.insert("raw_string".to_string(), to_raw_value(&str));
}
let hm = serde_json::from_str::<HashMap<String, Box<RawValue>>>(&str).map_err(|e| {
Error::BadRequest(format!("invalid cloudevents+json: {}", e)).into_response()
})?;
let hm = restructure_cloudevents_metadata(hm).map_err(|e| e.into_response())?;
Ok(PushArgsOwned { extra: Some(extra), args: hm })
}
}
#[cfg(test)]
mod tests {
use std::collections::HashMap;
use super::*;
#[tokio::test]
async fn test_cloudevents_json_payload() {
let r1 = r#"
{
"specversion" : "1.0",
"type" : "com.example.someevent",
"source" : "/mycontext",
"subject": null,
"id" : "C234-1234-1234",
"time" : "2018-04-05T17:31:00Z",
"comexampleextension1" : "value",
"comexampleothervalue" : 5,
"datacontenttype" : "application/json",
"data" : {
"appinfoA" : "abc",
"appinfoB" : 123,
"appinfoC" : true
}
}
"#;
let r2 = r#"
{
"specversion" : "1.0",
"type" : "com.example.someevent",
"source" : "/mycontext",
"subject": null,
"id" : "C234-1234-1234",
"time" : "2018-04-05T17:31:00Z",
"comexampleextension1" : "value",
"comexampleothervalue" : 5,
"datacontenttype" : "application/json",
"data" : 1.5
}
"#;
let extra = HashMap::new();
let a1 = PushArgsOwned::from_ce_json(extra.clone(), false, r1.to_string())
.await
.expect("Failed to parse the cloudevent");
let a2 = PushArgsOwned::from_ce_json(extra.clone(), false, r2.to_string())
.await
.expect("Failed to parse the cloudevent");
a1.args.get("WEBHOOK__METADATA__").expect(
"CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs",
);
assert_eq!(
a2.args
.get("body")
.expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs")
.to_string(),
"1.5"
);
}
}
backend/windmill-api/src/capture.rs
+8 -3
View File
@@ -18,9 +18,12 @@ use windmill_common::{
error::{JsonResult, Result},
utils::{not_found_if_none, StripPath},
};
use windmill_queue::{PushArgs, PushArgsOwned};
use windmill_queue::PushArgs;
use crate::db::{ApiAuthed, DB};
use crate::{
args::WebhookArgs,
db::{ApiAuthed, DB},
};
const KEEP_LAST: i64 = 8;
@@ -86,8 +89,10 @@ pub async fn new_payload(
pub async fn update_payload(
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
args: PushArgsOwned,
args: WebhookArgs,
) -> Result<StatusCode> {
let args = args.args;
let mut tx = db.begin().await?;
sqlx::query!(
backend/windmill-api/src/http_triggers.rs
+10 -2
View File
@@ -1,6 +1,7 @@
#[cfg(feature = "parquet")]
use crate::job_helpers_ee::get_workspace_s3_resource;
use crate::{
args::WebhookArgs,
db::{ApiAuthed, DB},
jobs::{
run_flow_by_path_inner, run_script_by_path_inner, run_wait_result_flow_by_path_internal,
@@ -32,7 +33,6 @@ use windmill_common::{
utils::{not_found_if_none, paginate, require_admin, Pagination, StripPath},
worker::{to_raw_value, CLOUD_HOSTED},
};
use windmill_queue::PushArgsOwned;
lazy_static::lazy_static! {
static ref ROUTE_PATH_KEY_RE: regex::Regex = regex::Regex::new(r"/:\w+").unwrap();
@@ -531,7 +531,7 @@ async fn route_job(
Query(query): Query<HashMap<String, String>>,
method: http::Method,
headers: HeaderMap,
mut args: PushArgsOwned,
args: WebhookArgs,
) -> impl IntoResponse {
let route_path = route_path.to_path();
let (trigger, called_path, params, authed) = match get_http_route_trigger(
@@ -547,6 +547,14 @@ async fn route_job(
Err(e) => return e.into_response(),
};
let mut args = match args
.to_push_args_owned(&authed, &db, &trigger.workspace_id)
.await
{
Ok(args) => args,
Err(e) => return e.into_response(),
};
#[cfg(not(feature = "parquet"))]
if trigger.static_asset_config.is_some() {
return error::Error::InternalErr(
backend/windmill-api/src/jobs.rs
+35 -13
View File
@@ -36,6 +36,7 @@ use crate::db::ApiAuthed;
use crate::users::get_scope_tags;
use crate::utils::content_plain;
use crate::{
args::{DecodeQueries, WebhookArgs},
db::DB,
users::{check_scopes, require_owner_of_path, OptAuthed},
utils::require_super_admin,
@@ -83,7 +84,7 @@ use windmill_common::{METRICS_DEBUG_ENABLED, METRICS_ENABLED};
use windmill_common::{get_latest_deployed_hash_for_path, BASE_URL};
use windmill_queue::{
cancel_job, get_queued_job, get_result_by_id_from_running_flow, job_is_complete, push,
DecodeQueries, PushArgs, PushArgsOwned, PushIsolationLevel,
PushArgs, PushArgsOwned, PushIsolationLevel,
};
#[cfg(feature = "prometheus")]
@@ -631,7 +632,10 @@ async fn get_flow_job_debug_info(
}
}
for job_id in job_ids {
let job = GetQuery::new().with_auth(&opt_authed).fetch(&db, job_id, &w_id).await;
let job = GetQuery::new()
.with_auth(&opt_authed)
.fetch(&db, job_id, &w_id)
.await;
if let Ok(job) = job {
jobs.insert(job.id().to_string(), job);
}
@@ -832,7 +836,10 @@ impl<'a> GetQuery<'a> {
self.check_auth(cjob.as_ref().map(|job| job.created_by.as_str()))?;
if self.with_flow {
cjob = resolve_maybe_value(db, workspace_id, self.with_code, cjob, |job| job.raw_flow.as_mut()).await?;
cjob = resolve_maybe_value(db, workspace_id, self.with_code, cjob, |job| {
job.raw_flow.as_mut()
})
.await?;
}
if let Some(mut cjob) = cjob {
cjob.inner = format_completed_job_result(cjob.inner);
@@ -842,14 +849,16 @@ impl<'a> GetQuery<'a> {
}
async fn fetch(self, db: &DB, job_id: Uuid, workspace_id: &str) -> error::Result<Job> {
let cjob = self.fetch_completed(db, job_id, workspace_id)
let cjob = self
.fetch_completed(db, job_id, workspace_id)
.await?
.map(Job::CompletedJob);
match cjob {
Some(cjob) => Ok(cjob),
None => {
let job_maybe = self.fetch_queued(db, job_id, workspace_id)
let job_maybe = self
.fetch_queued(db, job_id, workspace_id)
.await?
.map(Job::QueuedJob);
not_found_if_none(job_maybe, "Job", job_id.to_string())
@@ -2940,8 +2949,10 @@ pub async fn run_flow_by_path(
Extension(user_db): Extension<UserDB>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgsOwned,
args: WebhookArgs,
) -> error::Result<(StatusCode, String)> {
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
run_flow_by_path_inner(authed, db, user_db, w_id, flow_path, run_query, args, None).await
}
@@ -3116,8 +3127,9 @@ pub async fn run_script_by_path(
Extension(user_db): Extension<UserDB>,
Path((w_id, script_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgsOwned,
args: WebhookArgs,
) -> error::Result<(StatusCode, String)> {
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
run_script_by_path_inner(
authed,
db,
@@ -3218,7 +3230,10 @@ pub async fn run_workflow_as_code(
i += 1;
}
let job = GetQuery::new().without_logs().fetch_queued(&db, job_id, &w_id).await?;
let job = GetQuery::new()
.without_logs()
.fetch_queued(&db, job_id, &w_id)
.await?;
if *CLOUD_HOSTED {
tracing::info!("workflow_as_code_tracing id {i} ");
@@ -3779,11 +3794,13 @@ pub async fn run_wait_result_script_by_path(
Extension(db): Extension<DB>,
Path((w_id, script_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgsOwned,
args: WebhookArgs,
) -> error::Result<Response> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
run_wait_result_script_by_path_internal(
db,
run_query,
@@ -3861,11 +3878,13 @@ pub async fn run_wait_result_script_by_hash(
Extension(db): Extension<DB>,
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgsOwned,
args: WebhookArgs,
) -> error::Result<Response> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
check_queue_too_long(&db, run_query.queue_limit).await?;
let hash = script_hash.0;
@@ -3945,11 +3964,13 @@ pub async fn run_wait_result_flow_by_path(
Extension(db): Extension<DB>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgsOwned,
args: WebhookArgs,
) -> error::Result<Response> {
#[cfg(feature = "enterprise")]
check_license_key_valid().await?;
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
run_wait_result_flow_by_path_internal(
db, run_query, flow_path, authed, user_db, args, w_id, None,
)
@@ -4692,12 +4713,12 @@ async fn run_preview_flow_job(
pub async fn run_job_by_hash(
authed: ApiAuthed,
Extension(db): Extension<DB>,
Extension(user_db): Extension<UserDB>,
Path((w_id, script_hash)): Path<(String, ScriptHash)>,
Query(run_query): Query<RunJobQuery>,
args: PushArgsOwned,
args: WebhookArgs,
) -> error::Result<(StatusCode, String)> {
let args = args.to_push_args_owned(&authed, &db, &w_id).await?;
run_job_by_hash_inner(
authed,
db,
@@ -4747,6 +4768,7 @@ pub async fn run_job_by_hash_inner(
let tag = run_query.tag.clone().or(tag);
check_tag_available_for_workspace(&w_id, &tag, &authed).await?;
let tx = PushIsolationLevel::Isolated(user_db, authed.clone().into());
let (uuid, tx) = push(
backend/windmill-api/src/lib.rs
+1
View File
@@ -46,6 +46,7 @@ use windmill_common::error::AppError;
mod ai;
mod apps;
mod args;
mod audit;
mod capture;
mod concurrency_groups;
backend/windmill-api/src/tracing_init.rs
+8 -6
View File
@@ -29,11 +29,13 @@ impl<B> OnResponse<B> for MyOnResponse {
_span: &tracing::Span,
) {
if *LOG_REQUESTS {
tracing::info!(
latency = latency.as_millis(),
status = response.status().as_u16(),
"response"
)
let latency = latency.as_millis();
let status = response.status().as_u16();
if response.status().is_success() {
tracing::info!(latency = latency, status = status, "response")
} else {
tracing::error!(latency = latency, status = status, "response")
}
}
}
}
@@ -67,7 +69,7 @@ impl<B> MakeSpan<B> for MyMakeSpan {
uri = %request.uri(),
username = field::Empty,
workspace_id = field::Empty,
trace_id = tracing_id,
traceId = tracing_id,
email = field::Empty,
)
}
backend/windmill-api/src/users.rs
+21 -10
View File
@@ -847,6 +847,7 @@ pub struct UserInfo {
pub folders_read: Vec<String>,
pub folders: Vec<String>,
pub folders_owners: Vec<String>,
pub name: Option<String>,
}
#[derive(FromRow, Serialize)]
@@ -1218,6 +1219,7 @@ async fn whoami(
workspace_id: w_id,
email: email.clone(),
username: email,
name: None,
is_admin,
is_super_admin: is_admin,
created_at: chrono::Utc::now(),
@@ -1306,22 +1308,30 @@ async fn get_usage(
Ok(usage.to_string())
}
#[derive(FromRow, Serialize)]
pub struct User2 {
pub workspace_id: String,
pub email: String,
pub username: String,
pub is_admin: bool,
pub created_at: chrono::DateTime<chrono::Utc>,
pub operator: bool,
pub disabled: bool,
pub role: Option<String>,
pub super_admin: bool,
pub name: Option<String>,
}
async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo>> {
let user = sqlx::query_as!(
User,
"SELECT * FROM usr where username = $1 AND workspace_id = $2",
User2,
"SELECT usr.*, password.super_admin, password.name FROM usr LEFT JOIN password ON usr.email = password.email Where usr.username = $1 AND workspace_id = $2
",
username,
w_id
)
.fetch_optional(db)
.await?;
let is_super_admin = sqlx::query_scalar!(
"SELECT super_admin FROM password WHERE email = $1",
user.as_ref().map(|x| &x.email)
)
.fetch_optional(db)
.await?
.unwrap_or(false);
let groups = get_groups_for_user(
&w_id,
username,
@@ -1339,8 +1349,9 @@ async fn get_user(w_id: &str, username: &str, db: &DB) -> Result<Option<UserInfo
workspace_id: usr.workspace_id,
email: usr.email,
username: usr.username,
name: usr.name,
is_admin: usr.is_admin,
is_super_admin,
is_super_admin: usr.super_admin,
created_at: usr.created_at,
operator: usr.operator,
disabled: usr.disabled,
backend/windmill-common/Cargo.toml
+10
View File
@@ -13,6 +13,8 @@ flamegraph = ["dep:tracing-flame"]
loki = ["dep:tracing-loki"]
benchmark = []
parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"]
otel = ["dep:opentelemetry-semantic-conventions", "dep:opentelemetry-otlp", "dep:opentelemetry_sdk",
"dep:opentelemetry", "dep:tracing-opentelemetry", "dep:opentelemetry-appender-tracing", "dep:tonic"]
[lib]
name = "windmill_common"
@@ -64,5 +66,13 @@ croner = "2.0.6"
quick_cache.workspace = true
pin-project-lite.workspace = true
opentelemetry-semantic-conventions = { workspace = true, optional = true }
opentelemetry-otlp = { workspace = true, optional = true }
opentelemetry_sdk = { workspace = true, optional = true }
opentelemetry = { workspace = true, optional = true }
tracing-opentelemetry = { workspace = true, optional = true }
opentelemetry-appender-tracing = { workspace = true, optional = true }
tonic = { workspace = true, optional = true }
[target.'cfg(not(target_env = "msvc"))'.dependencies]
tikv-jemalloc-ctl = { optional = true, workspace = true }
backend/windmill-common/src/apps.rs
+12 -1
View File
@@ -6,7 +6,18 @@
* LICENSE-AGPL for a copy of the license.
*/
use serde::Deserialize;
use serde::{Deserialize, Serialize};
/// Id in the `app_script` table.
#[derive(Serialize, Deserialize, Debug, Copy, Clone, Hash, Eq, PartialEq)]
#[serde(transparent)]
pub struct AppScriptId(pub i64);
impl Into<u64> for AppScriptId {
fn into(self) -> u64 {
self.0 as u64
}
}
#[derive(Deserialize)]
pub struct ListAppQuery {
backend/windmill-common/src/cache.rs
+291 -64
View File
@@ -1,25 +1,149 @@
use crate::error;
use std::future::Future;
use std::hash::Hash;
use std::path::{Path, PathBuf};
use quick_cache::sync::Cache;
use quick_cache::Equivalent;
use serde::{Deserialize, Serialize};
use sqlx::PgExecutor;
pub use const_format::concatcp;
pub use lazy_static::lazy_static;
pub use quick_cache::sync::Cache;
/// Cache directory for windmill server/worker(s).
pub const CACHE_DIR: &str = "/tmp/windmill/cache/";
/// A file-system backed concurrent cache.
pub struct FsBackedCache<Key, Val> {
cache: Cache<Key, Val>,
root: &'static str,
}
impl<Key: Eq + Hash, Val: Clone + fs::Bundle> FsBackedCache<Key, Val> {
/// Create a new file-system backed cache with `items_capacity` capacity.
/// The cache will be stored in the `root` directory.
pub fn new(root: &'static str, items_capacity: usize) -> Self {
Self { cache: Cache::new(items_capacity), root }
}
/// Gets or inserts an item in the cache with key `key`.
pub async fn get_or_insert_async<'a, Q, F>(&'a self, key: &Q, with: F) -> error::Result<Val>
where
Q: Hash + Equivalent<Key> + ToOwned<Owned = Key> + Copy + Into<u64>,
F: Future<Output = error::Result<Val>>,
{
self.cache
.get_or_insert_async(
key,
fs::import_or_insert_with(self.root, (*key).into(), with),
)
.await
}
}
/// Like [`lazy_static`]`, but for file-system backed caches.
///
/// # Example
/// ```rust
/// use windmill_common::make_static;
///
/// make_static! {
/// /// String cache with a maximum capacity of 1000 items stored in the
/// /// "subdirectory" directory.
/// static ref CACHE: { u64 => String } in "subdirectory" <= 1000;
/// /// Another cache.
/// static ref ANOTHER_CACHE: { u64 => Vec<String> } in "another" <= 100;
/// }
/// ```
#[macro_export]
macro_rules! make_static {
{ $( $(#[$attr:meta])* static ref $name:ident: { $Key:ty => $Val:ty } in $root:literal <= $cap:literal; )+ } => {
$crate::cache::lazy_static! {
$(
$(#[$attr])*
static ref $name: $crate::cache::FsBackedCache<$Key, $Val> =
$crate::cache::FsBackedCache::new(
$crate::cache::concatcp!($crate::cache::CACHE_DIR, $root),
$cap
);
)+
}
};
}
// re-export:
pub use make_static;
/// Create an anonymous file-system backed cache for one-time use.
///
/// # Example
/// ```rust
/// use windmill_common::anon;
/// let cache = anon!({ u64 => String } in "subdirectory" <= 1000);
/// ```
#[macro_export]
macro_rules! anon {
({ $Key:ty => $Val:ty } in $root:literal <= $cap:literal) => {{
$crate::cache::make_static! {
static ref __ANON__: { $Key => $Val } in $root <= $cap;
}
&__ANON__
}};
}
// re-export:
pub use anon;
pub mod future {
use super::*;
/// Extension trait for futures that can be cached.
pub trait FutureCachedExt<Val: Clone + fs::Bundle>:
Future<Output = error::Result<Val>> + Sized
{
/// Get or insert the future result in the cache.
///
/// # Example
/// ```rust
/// use windmill_common::cache::{self, future::FutureCachedExt};
///
/// async {
/// let result = std::future::ready(Ok(42))
/// .cached(cache::anon!({ u64 => u64 } in "test" <= 1), &42)
/// .await;
///
/// assert_eq!(result.unwrap(), 42);
/// };
/// ```
fn cached<Key: Eq + Hash, Q>(
self,
cache: &FsBackedCache<Key, Val>,
key: &Q,
) -> impl Future<Output = error::Result<Val>>
where
Q: Hash + Equivalent<Key> + ToOwned<Owned = Key> + Copy + Into<u64>,
{
cache.get_or_insert_async(key, self)
}
}
impl<Val: Clone + fs::Bundle, F: Future<Output = error::Result<Val>> + Sized>
FutureCachedExt<Val> for F
{
}
}
pub mod flow {
use super::*;
use crate::flows::{FlowNodeId, FlowValue};
/// Cache directory for windmill server/worker(s) flow nodes.
pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "flow");
lazy_static::lazy_static! {
make_static! {
/// Flow node cache.
/// FIXME: This should be a static but [`Cache`] does not have a const constructor.
/// FIXME: Use `Arc<Val>` for cheap cloning.
static ref CACHE: Cache<FlowNodeId, Val> = Cache::new(1000);
static ref CACHE: { FlowNodeId => Val } in "flow" <= 1000;
}
/// Flow node cache value.
@@ -75,38 +199,35 @@ pub mod flow {
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
CACHE
.get_or_insert_async(
&node,
fs::import_or_insert_with(CACHE_DIR, node.0 as u64, async {
sqlx::query!(
"SELECT \
.get_or_insert_async(&node, async {
sqlx::query!(
"SELECT \
lock AS \"lock: String\", \
code AS \"code: String\", \
flow::text AS \"flow: Box<str>\" \
FROM flow_node WHERE id = $1 LIMIT 1",
node.0,
)
.fetch_one(e)
.await
.map_err(Into::into)
.and_then(|r| {
Ok(Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
flow: match r.flow {
None => None,
Some(flow) => serde_json::from_str(&flow).map_err(|err| {
error::Error::InternalErr(format!(
"Unable to parse flow value: {err:?}"
))
})?,
},
})
node.0,
)
.fetch_one(e)
.await
.map_err(Into::into)
.and_then(|r| {
Ok(Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
flow: match r.flow {
None => None,
Some(flow) => serde_json::from_str(&flow).map_err(|err| {
error::Error::InternalErr(format!(
"Unable to parse flow value: {err:?}"
))
})?,
},
})
}),
)
})
})
.await
}
@@ -164,14 +285,10 @@ pub mod script {
use super::*;
use crate::scripts::{ScriptHash, ScriptLang};
/// Cache directory for windmill server/worker(s) scripts.
pub const CACHE_DIR: &str = const_format::concatcp!(super::CACHE_DIR, "script");
lazy_static::lazy_static! {
make_static! {
/// Scripts cache.
/// FIXME: This should be a static but [`Cache`] does not have a const constructor.
/// FIXME: Use `Arc<Val>` for cheap cloning.
static ref CACHE: Cache<ScriptHash, Val> = Cache::new(1000);
static ref CACHE: { ScriptHash => Val } in "script" <= 1000;
}
/// Script cache value.
@@ -197,34 +314,31 @@ pub mod script {
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
CACHE
.get_or_insert_async(
&hash,
fs::import_or_insert_with(CACHE_DIR, hash.0 as u64, async {
sqlx::query!(
"SELECT \
.get_or_insert_async(&hash, async {
sqlx::query!(
"SELECT \
lock AS \"lock: String\", \
content AS \"code!: String\",
language AS \"language: Option<ScriptLang>\", \
envs AS \"envs: Vec<String>\", \
codebase AS \"codebase: String\" \
FROM script WHERE hash = $1 AND workspace_id = $2 LIMIT 1",
hash.0,
workspace_id,
)
.fetch_one(e)
.await
.map_err(Into::into)
.map(|r| Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
language: r.language,
envs: r.envs,
codebase: r.codebase,
})
}),
)
hash.0,
workspace_id,
)
.fetch_one(e)
.await
.map_err(Into::into)
.map(|r| Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
language: r.language,
envs: r.envs,
codebase: r.codebase,
})
})
.await
}
@@ -280,11 +394,99 @@ pub mod script {
}
}
pub mod app {
use super::*;
use crate::apps::AppScriptId;
make_static! {
/// App scripts cache.
/// FIXME: Use `Arc<Val>` for cheap cloning.
static ref CACHE: { AppScriptId => Val } in "app" <= 1000;
}
/// App app script cache value.
#[derive(Debug, Clone, Default)]
pub struct Val {
pub lock: Option<String>,
pub code: String,
}
/// Fetch the app script referenced by `id` from the cache.
/// If not present, import from the file-system cache or fetch it from the database and write
/// it to the file system and cache.
/// This should be preferred over fetching the database directly.
pub async fn fetch_script(
e: impl PgExecutor<'_>,
id: AppScriptId,
) -> error::Result<(Option<String>, String)> {
// If not present, `get_or_insert_async` will lock the key until the future completes,
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
CACHE
.get_or_insert_async(&id, async {
sqlx::query!(
"SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1",
id.0,
)
.fetch_one(e)
.await
.map_err(Into::into)
.map(|r| Val {
lock: r
.lock
.and_then(|x| if x.is_empty() { None } else { Some(x) }),
code: r.code,
})
})
.await
.map(|Val { lock, code }| (lock, code))
}
// ----------------------------------------------------------------------------------------------
// impl `fs::Bundle` for `Val`.
#[derive(Copy, Clone)]
pub enum Item {
Lock,
Code,
}
impl fs::Item for Item {
fn path(&self, root: &Path) -> PathBuf {
match self {
Item::Lock => root.join("lock.txt"),
Item::Code => root.join("code.txt"),
}
}
}
impl fs::Bundle for Val {
type Item = Item;
fn items() -> &'static [Self::Item] {
&[Item::Lock, Item::Code]
}
fn import(&mut self, item: Self::Item, data: Vec<u8>) -> error::Result<()> {
match item {
Item::Lock => self.lock = Some(String::from_utf8(data)?),
Item::Code => self.code = String::from_utf8(data)?,
}
Ok(())
}
fn export(&self, item: Self::Item) -> error::Result<Option<Vec<u8>>> {
match item {
Item::Lock => Ok(self.lock.as_ref().map(|s| s.as_bytes().to_vec())),
Item::Code => Ok(Some(self.code.as_bytes().to_vec())),
}
}
}
}
mod fs {
use super::*;
use std::future::Future;
use std::fs::{self, OpenOptions};
use std::io::{Read, Write};
@@ -363,4 +565,29 @@ mod fs {
}
Ok(data)
}
// Auto-implement `Bundle` for all `serde` serializable types.
impl Item for () {
fn path(&self, root: &Path) -> PathBuf {
root.join("self.json")
}
}
impl<T: for<'de> Deserialize<'de> + Serialize + Default> Bundle for T {
type Item = ();
fn items() -> &'static [Self::Item] {
&[()]
}
fn import(&mut self, _: Self::Item, data: Vec<u8>) -> error::Result<()> {
*self = serde_json::from_slice(&data)?;
Ok(())
}
fn export(&self, _: Self::Item) -> error::Result<Option<Vec<u8>>> {
Ok(Some(serde_json::to_vec(self)?))
}
}
}
backend/windmill-common/src/db.rs
+8 -8
View File
@@ -80,14 +80,14 @@ impl UserDB {
let mut folders_read = folders_read.clone();
folders_read.extend(folders_write.clone());
tracing::debug!(
"Setting role to {} {:?} {:?} {:?} {:?}",
user,
authed.username(),
authed.groups(),
folders_read,
folders_write
);
// tracing::debug!(
// "Setting role to {} {:?} {:?} {:?} {:?}",
// user,
// authed.username(),
// authed.groups(),
// folders_read,
// folders_write
// );
let mut tx = self.db.begin().await?;
backend/windmill-common/src/flow_status.rs
+2 -2
View File
@@ -17,7 +17,7 @@ use crate::flows::FlowValue;
const MINUTES: Duration = Duration::from_secs(60);
const HOURS: Duration = MINUTES.saturating_mul(60);
pub const MAX_RETRY_ATTEMPTS: u16 = 1000;
pub const MAX_RETRY_ATTEMPTS: u32 = u32::MAX;
pub const MAX_RETRY_INTERVAL: Duration = HOURS.saturating_mul(6);
pub fn is_retry_default(v: &RetryStatus) -> bool {
@@ -48,7 +48,7 @@ pub struct FlowStatus {
#[derive(Serialize, Deserialize, Debug, Clone, Default)]
#[serde(default)]
pub struct RetryStatus {
pub fail_count: u16,
pub fail_count: u32,
pub failed_jobs: Vec<Uuid>,
}
backend/windmill-common/src/flows.rs
+75 -23
View File
@@ -147,7 +147,7 @@ impl Retry {
/// Takes the number of previous retries and returns the interval until the next retry if any.
///
/// May return [`Duration::ZERO`] to retry immediately.
pub fn interval(&self, previous_attempts: u16, silent: bool) -> Option<Duration> {
pub fn interval(&self, previous_attempts: u32, silent: bool) -> Option<Duration> {
let Self { constant, exponential } = self;
if previous_attempts < constant.attempts {
@@ -178,7 +178,7 @@ impl Retry {
self.constant.attempts != 0 || self.exponential.attempts != 0
}
pub fn max_attempts(&self) -> u16 {
pub fn max_attempts(&self) -> u32 {
self.constant
.attempts
.saturating_add(self.exponential.attempts)
@@ -194,7 +194,7 @@ impl Retry {
#[derive(Deserialize, Serialize, Debug, Clone, Default, PartialEq)]
#[serde(default)]
pub struct ConstantDelay {
pub attempts: u16,
pub attempts: u32,
pub seconds: u16,
}
@@ -202,7 +202,7 @@ pub struct ConstantDelay {
#[derive(Deserialize, Serialize, Debug, Clone, PartialEq)]
#[serde(default)]
pub struct ExponentialDelay {
pub attempts: u16,
pub attempts: u32,
pub multiplier: u16,
pub seconds: u16,
pub random_factor: Option<i8>, // percentage, defaults to 0 for no jitter
@@ -407,6 +407,12 @@ pub enum InputTransform {
#[serde(transparent)]
pub struct FlowNodeId(pub i64);
impl Into<u64> for FlowNodeId {
fn into(self) -> u64 {
self.0 as u64
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct Branch {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -713,10 +719,14 @@ pub async fn resolve_maybe_value<T>(
workspace_id: &str,
with_code: bool,
maybe: Option<T>,
value_mut: impl FnOnce(&mut T) -> Option<&mut Json<Box<JsonRawValue>>>
value_mut: impl FnOnce(&mut T) -> Option<&mut Json<Box<JsonRawValue>>>,
) -> Result<Option<T>, Error> {
let Some(mut container) = maybe else { return Ok(None); };
let Some(value) = value_mut(&mut container) else { return Ok(Some(container)); };
let Some(mut container) = maybe else {
return Ok(None);
};
let Some(value) = value_mut(&mut container) else {
return Ok(Some(container));
};
resolve_value(e, workspace_id, &mut value.0, with_code).await?;
Ok(Some(container))
}
@@ -728,8 +738,9 @@ pub async fn resolve_value(
value: &mut Box<JsonRawValue>,
with_code: bool,
) -> Result<(), Error> {
let mut val = serde_json::from_str::<FlowValue>(value.get())
.map_err(|err| Error::InternalErr(format!("resolve: Failed to parse flow value: {}", err)))?;
let mut val = serde_json::from_str::<FlowValue>(value.get()).map_err(|err| {
Error::InternalErr(format!("resolve: Failed to parse flow value: {}", err))
})?;
for module in &mut val.modules {
resolve_module(e, workspace_id, &mut module.value, with_code).await?;
}
@@ -746,16 +757,29 @@ pub async fn resolve_module(
) -> Result<(), Error> {
use FlowModuleValue::*;
let mut val = serde_json::from_str::<FlowModuleValue>(value.get())
.map_err(|err| Error::InternalErr(format!("resolve: Failed to parse flow module value: {}", err)))?;
let mut val = serde_json::from_str::<FlowModuleValue>(value.get()).map_err(|err| {
Error::InternalErr(format!(
"resolve: Failed to parse flow module value: {}",
err
))
})?;
match &mut val {
FlowScript { .. } => {
// In order to avoid an unnecessary `.clone()` of `val`, take ownership of it's content
// using `std::mem::replace`.
let FlowScript {
input_transforms, id, tag, language,
custom_concurrency_key, concurrent_limit, concurrency_time_window_s, is_trigger
} = std::mem::replace(&mut val, Identity) else { unreachable!() };
input_transforms,
id,
tag,
language,
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
is_trigger,
} = std::mem::replace(&mut val, Identity)
else {
unreachable!()
};
// Load script lock file and code content.
let (lock, content) = if !with_code {
(Some("...".to_string()), "...".to_string())
@@ -763,22 +787,44 @@ pub async fn resolve_module(
cache::flow::fetch_script(e, id).await?
};
val = RawScript {
input_transforms, content, lock, path: None, tag, language, custom_concurrency_key,
concurrent_limit, concurrency_time_window_s, is_trigger
input_transforms,
content,
lock,
path: None,
tag,
language,
custom_concurrency_key,
concurrent_limit,
concurrency_time_window_s,
is_trigger,
};
},
ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => {
}
ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => {
resolve_modules(e, workspace_id, modules, modules_node.take(), with_code).await?;
},
}
BranchOne { branches, default, default_node } => {
resolve_modules(e, workspace_id, default, default_node.take(), with_code).await?;
for branch in branches {
resolve_modules(e, workspace_id, &mut branch.modules, branch.modules_node.take(), with_code).await?;
resolve_modules(
e,
workspace_id,
&mut branch.modules,
branch.modules_node.take(),
with_code,
)
.await?;
}
},
}
BranchAll { branches, .. } => {
for branch in branches {
resolve_modules(e, workspace_id, &mut branch.modules, branch.modules_node.take(), with_code).await?;
resolve_modules(
e,
workspace_id,
&mut branch.modules,
branch.modules_node.take(),
with_code,
)
.await?;
}
}
_ => {}
@@ -801,7 +847,13 @@ pub async fn resolve_modules(
.map(|flow| flow.modules)?;
}
for module in modules.iter_mut() {
Box::pin(resolve_module(e, workspace_id, &mut module.value, with_code)).await?;
Box::pin(resolve_module(
e,
workspace_id,
&mut module.value,
with_code,
))
.await?;
}
Ok(())
}
backend/windmill-common/src/global_settings.rs
+5 -1
View File
@@ -35,8 +35,9 @@ pub const CRITICAL_ALERT_MUTE_UI_SETTING: &str = "critical_alert_mute_ui";
pub const DEV_INSTANCE_SETTING: &str = "dev_instance";
pub const JWT_SECRET_SETTING: &str = "jwt_secret";
pub const EMAIL_DOMAIN_SETTING: &str = "email_domain";
pub const OTEL_SETTING: &str = "otel";
pub const ENV_SETTINGS: [&str; 51] = [
pub const ENV_SETTINGS: [&str; 54] = [
"DISABLE_NSJAIL",
"MODE",
"NUM_WORKERS",
@@ -88,4 +89,7 @@ pub const ENV_SETTINGS: [&str; 51] = [
"WORKER_GROUP",
"SAML_METADATA",
"INSTANCE_IS_DEV",
"OTEL_METRICS",
"OTEL_TRACING",
"OTEL_LOGS",
];
backend/windmill-common/src/jobs.rs
+8
View File
@@ -14,6 +14,7 @@ pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE";
pub const PREPROCESSOR_FAKE_ENTRYPOINT: &str = "__WM_PREPROCESSOR";
use crate::{
apps::AppScriptId,
error::{self, to_anyhow, Error},
flow_status::{FlowStatus, RestartedFrom},
flows::{FlowNodeId, FlowValue, Retry},
@@ -41,6 +42,7 @@ pub enum JobKind {
DeploymentCallback,
FlowScript,
FlowNode,
AppScript,
}
#[derive(sqlx::FromRow, Debug, Serialize, Clone)]
@@ -278,6 +280,12 @@ pub enum JobPayload {
id: FlowNodeId, // flow_node(id).
path: String, // flow node inner path (e.g. `outer/branchall-42`).
},
AppScript {
id: AppScriptId, // app_script(id).
path: Option<String>,
language: ScriptLang,
cache_ttl: Option<i32>,
},
Code(RawCode),
Dependencies {
path: String,
backend/windmill-common/src/lib.rs
+8 -2
View File
@@ -36,20 +36,20 @@ pub mod job_s3_helpers_ee;
pub mod jobs;
pub mod more_serde;
pub mod oauth2;
pub mod otel_ee;
pub mod queue;
pub mod s3_helpers;
pub mod schedule;
pub mod scripts;
pub mod server;
pub mod stats_ee;
pub mod tracing_init;
pub mod users;
pub mod utils;
pub mod variables;
pub mod worker;
pub mod workspaces;
pub mod tracing_init;
pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5;
pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5;
@@ -87,6 +87,12 @@ lazy_static::lazy_static! {
.unwrap_or_else(|| SocketAddr::from(([0, 0, 0, 0], *METRICS_PORT)));
pub static ref METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("METRICS_PORT").is_ok() || std::env::var("METRICS_ADDR").is_ok());
pub static ref OTEL_METRICS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_METRICS").is_ok());
pub static ref OTEL_TRACING_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_TRACING").is_ok());
pub static ref OTEL_LOGS_ENABLED: AtomicBool = AtomicBool::new(std::env::var("OTEL_LOGS").is_ok());
pub static ref METRICS_DEBUG_ENABLED: AtomicBool = AtomicBool::new(false);
pub static ref CRITICAL_ALERT_MUTE_UI_ENABLED: AtomicBool = AtomicBool::new(false);
backend/windmill-common/src/otel_ee.rs
+58
View File
@@ -0,0 +1,58 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2022
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{jobs::QueuedJob, utils::Mode};
use uuid::Uuid;
pub fn set_span_parent(_span: &tracing::Span, _rj: &Uuid) {}
#[cfg(not(all(feature = "otel", feature = "enterprise")))]
pub(crate) type OtelProvider = Option<()>;
#[cfg(all(feature = "otel", feature = "enterprise"))]
pub(crate) type OtelProvider = Option<opentelemetry_sdk::metrics::SdkMeterProvider>;
#[cfg(not(feature = "otel"))]
pub fn otel_ctx() -> () {}
#[cfg(feature = "otel")]
#[inline(always)]
pub fn otel_ctx() -> opentelemetry::Context {
opentelemetry::Context::current()
}
#[cfg(not(feature = "otel"))]
impl<T: Sized> FutureExt for T {}
#[cfg(not(feature = "otel"))]
pub trait FutureExt: Sized {
fn with_context(self, _otel_cx: ()) -> Self {
self
}
}
use tracing_subscriber::EnvFilter;
pub(crate) fn init_logs_bridge(_mode: &Mode, _hostname: &str, _env: &str) -> Option<EnvFilter> {
None
}
#[cfg(all(feature = "otel", feature = "enterprise"))]
pub(crate) fn init_otlp_tracer(
_mode: &Mode,
_hostname: &str,
_env: &str,
) -> Option<opentelemetry_sdk::trace::Tracer> {
None
}
pub(crate) fn init_meter_provider(_mode: &Mode, _hostname: &str, _env: &str) -> OtelProvider {
None
}
pub fn add_root_flow_job_to_otlp(_queued_job: &QueuedJob, _success: bool) {}
backend/windmill-common/src/s3_helpers.rs
+1 -1
View File
@@ -122,7 +122,7 @@ pub struct S3Object {
#[cfg(feature = "parquet")]
pub async fn get_etag_or_empty(
object_store_resource: &mut ObjectStoreResource,
object_store_resource: &ObjectStoreResource,
s3_object: S3Object,
) -> Option<String> {
let object_store_client = build_object_store_client(object_store_resource).await;
backend/windmill-common/src/scripts.rs
+6
View File
@@ -77,6 +77,12 @@ impl ScriptLang {
#[sqlx(transparent)]
pub struct ScriptHash(pub i64);
impl Into<u64> for ScriptHash {
fn into(self) -> u64 {
self.0 as u64
}
}
#[derive(PartialEq, sqlx::Type)]
#[sqlx(transparent, no_pg_array)]
pub struct ScriptHashes(pub Vec<i64>);
backend/windmill-common/src/tracing_init.rs
+69 -39
View File
@@ -7,13 +7,23 @@
*/
use const_format::concatcp;
use std::{
collections::HashMap,
sync::{Arc, RwLock},
};
use tracing::{level_filters::LevelFilter, Event};
use tracing_appender::non_blocking::{NonBlockingBuilder, WorkerGuard};
use tracing_subscriber::layer::Context;
use tracing_subscriber::{
filter::Targets,
fmt::{format, Layer},
prelude::*,
EnvFilter,
};
use crate::utils::Mode;
fn json_layer<S>() -> Layer<S, format::JsonFields, format::Format<format::Json>> {
tracing_subscriber::fmt::layer()
.json()
@@ -34,17 +44,40 @@ pub const LOGS_SERVICE: &str = "logs/services/";
pub const TMP_WINDMILL_LOGS_SERVICE: &str = concatcp!("/tmp/windmill/", LOGS_SERVICE);
pub fn initialize_tracing(hostname: &str) -> WorkerGuard {
pub fn initialize_tracing(
hostname: &str,
mode: &Mode,
environment: &str,
) -> (WorkerGuard, crate::otel_ee::OtelProvider) {
let style = std::env::var("RUST_LOG_STYLE").unwrap_or_else(|_| "auto".into());
if std::env::var("RUST_LOG").is_ok_and(|x| x == "debug" || x == "info") {
let rust_log_env = std::env::var("RUST_LOG");
if rust_log_env
.as_ref()
.is_ok_and(|x| x == "debug" || x == "info")
{
std::env::set_var(
"RUST_LOG",
&format!("windmill={}", std::env::var("RUST_LOG").unwrap()),
&format!("windmill={}", rust_log_env.as_ref().unwrap()),
)
}
let default_env_filter = if rust_log_env.is_ok_and(|x| x == "debug") {
LevelFilter::DEBUG
} else {
LevelFilter::INFO
};
let meter_provider = crate::otel_ee::init_meter_provider(mode, hostname, environment);
#[cfg(all(feature = "otel", feature = "enterprise"))]
let opentelemetry = crate::otel_ee::init_otlp_tracer(mode, hostname, environment)
.map(|x| tracing_opentelemetry::layer().with_tracer(x));
#[cfg(not(all(feature = "otel", feature = "enterprise")))]
let opentelemetry: Option<EnvFilter> = None;
let logs_bridge = crate::otel_ee::init_logs_bridge(&mode, hostname, environment);
let env_filter = EnvFilter::from_default_env();
use tracing_appender::rolling::{RollingFileAppender, Rotation};
let log_dir = format!("{}/{}/", TMP_WINDMILL_LOGS_SERVICE, hostname);
@@ -61,39 +94,59 @@ pub fn initialize_tracing(hostname: &str) -> WorkerGuard {
.finish(file_appender);
let stdout_and_log_file_writer = std::io::stdout.and(log_file_writer);
let ts_base = tracing_subscriber::registry().with(env_filter);
// let job_logs_filter = tracing_subscriber::filter::Targets::new()
// .with_target("windmill:job_log", tracing::Level::TRACE);
#[cfg(feature = "loki")]
let ts_base = {
let (layer, task) = tracing_loki::builder()
.build_url(reqwest::Url::parse("http://127.0.0.1:3100").unwrap())
.expect("build loki url");
tokio::spawn(task);
ts_base.with(layer)
};
let env_filter = EnvFilter::builder()
.with_default_directive(tracing::level_filters::LevelFilter::ERROR.into())
.from_env_lossy();
let ts_base = tracing_subscriber::registry().with(env_filter);
match *JSON_FMT {
true => ts_base
.with(logs_bridge)
.with(opentelemetry)
// .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap()))
.with(
json_layer()
.with_writer(stdout_and_log_file_writer)
.flatten_event(true),
.flatten_event(true)
.with_filter(
Targets::new()
.with_target(
"windmill:job_log",
tracing::level_filters::LevelFilter::OFF,
)
.with_default(default_env_filter),
),
)
.with(CountingLayer::new())
.init(),
false => ts_base
.with(logs_bridge)
.with(opentelemetry)
// .with(env_filter2.add_directive("windmill:job_log=off".parse().unwrap()))
.with(
compact_layer()
.with_writer(stdout_and_log_file_writer)
.with_ansi(style.to_lowercase() != "never")
.with_file(true)
.with_line_number(true)
.with_target(false),
.with_target(false)
.with_filter(
Targets::new()
.with_target(
"windmill:job_log",
tracing::level_filters::LevelFilter::OFF,
)
.with_default(default_env_filter),
),
)
.with(CountingLayer::new())
.init(),
}
_guard
(_guard, meter_provider)
}
#[cfg(feature = "flamegraph")]
@@ -112,13 +165,6 @@ pub fn setup_flamegraph() -> impl Drop {
_guard
}
use std::{
collections::HashMap,
sync::{Arc, RwLock},
};
use tracing::Event;
use tracing_subscriber::layer::Context;
lazy_static::lazy_static! {
pub static ref LOG_COUNTING_BY_MIN: Arc<RwLock<HashMap<String, LogCounter>>> = Arc::new(RwLock::new(HashMap::new()));
}
@@ -144,22 +190,6 @@ impl CountingLayer {
}
}
// impl CountingLayer {
// pub fn new() -> Self {
// CountingLayer { counter: Arc::new(Mutex::new(LogCounter::new())) }
// }
// pub fn get_counts(&self) -> (usize, usize) {
// let counter = self.counter.lock().unwrap();
// (counter.non_error_count, counter.error_count)
// }
// pub fn reset_counts(&self) {
// let mut counter = self.counter.lock().unwrap();
// counter.reset();
// }
// }
pub const LOG_TIMESTAMP_FMT: &str = "%Y-%m-%d-%H-%M";
impl<S> tracing_subscriber::Layer<S> for CountingLayer
backend/windmill-common/src/worker.rs
+12 -3
View File
@@ -17,7 +17,9 @@ use std::{
use tokio::sync::RwLock;
use windmill_macros::annotations;
use crate::{error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB};
use crate::{
error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp, DB,
};
lazy_static::lazy_static! {
pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
@@ -353,6 +355,11 @@ pub struct SqlAnnotations {
pub return_last_result: bool,
}
#[annotations("#")]
pub struct BashAnnotations {
pub docker: bool,
}
pub async fn load_cache(bin_path: &str, _remote_path: &str) -> (bool, String) {
if tokio::fs::metadata(&bin_path).await.is_ok() {
(true, format!("loaded from local cache: {}\n", bin_path))
@@ -578,8 +585,10 @@ pub fn get_windmill_memory_usage() -> Option<i64> {
}
}
pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(executor: E) -> bool {
use crate::utils::{GIT_VERSION, GIT_SEM_VERSION};
pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(
executor: E,
) -> bool {
use crate::utils::{GIT_SEM_VERSION, GIT_VERSION};
// fetch all pings with a different version than self from the last 5 minutes.
let pings = sqlx::query_scalar!(
backend/windmill-queue/Cargo.toml
+2 -1
View File
@@ -43,4 +43,5 @@ bigdecimal.workspace = true
axum.workspace = true
serde_urlencoded.workspace = true
regex.workspace = true
backon.workspace = true
backon.workspace = true
opentelemetry.workspace = true
backend/windmill-queue/src/jobs.rs
+23 -353
View File
@@ -10,29 +10,19 @@ use std::{borrow::Borrow, collections::HashMap, sync::Arc, vec};
use anyhow::Context;
use async_recursion::async_recursion;
use axum::{
body::Bytes,
extract::{FromRequest, FromRequestParts, Query},
http::{request::Parts, Request, Uri},
response::{IntoResponse, Response},
};
use chrono::{DateTime, Duration, Utc};
use futures::future::TryFutureExt;
use itertools::Itertools;
#[cfg(feature = "prometheus")]
use prometheus::IntCounter;
use regex::Regex;
use reqwest::{
header::{HeaderMap, CONTENT_TYPE},
Client, StatusCode,
};
use serde::{ser::SerializeMap, Deserialize, Serialize};
use reqwest::Client;
use serde::{ser::SerializeMap, Serialize};
use serde_json::{json, value::RawValue};
use sqlx::{types::Json, FromRow, Pool, Postgres, Transaction};
#[cfg(feature = "benchmark")]
use std::time::Instant;
use tokio::{sync::RwLock, time::sleep};
use tracing::{instrument, Instrument};
use ulid::Ulid;
use uuid::Uuid;
use windmill_audit::audit_ee::{audit_log, AuditAuthor};
@@ -454,7 +444,6 @@ where
}
}
#[instrument(level = "trace", skip_all)]
pub async fn add_completed_job_error(
db: &Pool<Postgres>,
queued_job: &QueuedJob,
@@ -510,7 +499,6 @@ lazy_static::lazy_static! {
pub static ref GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE: Option<String> = std::env::var("GLOBAL_ERROR_HANDLER_PATH_IN_ADMINS_WORKSPACE").ok();
}
#[instrument(level = "trace", skip_all, name = "add_completed_job")]
pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
db: &Pool<Postgres>,
queued_job: &QueuedJob,
@@ -643,9 +631,7 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
.fetch_one(&mut *tx)
.await
.map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e:#}")))?;
// tracing::error!("2 {:?}", start.elapsed());
// add_time!(bench, "add_completed_job query END");
if !queued_job.is_flow_step {
if _duration > 500
@@ -1259,7 +1245,6 @@ pub async fn send_error_to_workspace_handler<'a, 'c, T: Serialize + Send + Sync>
Ok(())
}
#[instrument(level = "trace", skip_all)]
pub async fn handle_maybe_scheduled_job<'c>(
db: &Pool<Postgres>,
job: &QueuedJob,
@@ -2429,7 +2414,6 @@ async fn extract_result_from_job_result(
}
}
#[instrument(level = "trace", skip_all)]
pub async fn delete_job<'c>(
mut tx: Transaction<'c, Postgres>,
w_id: &str,
@@ -2576,340 +2560,6 @@ impl<'c> Serialize for PushArgs<'c> {
}
}
#[derive(Deserialize)]
pub struct DecodeQuery {
pub include_query: Option<String>,
}
#[derive(Deserialize)]
pub struct IncludeQuery {
pub include_query: Option<String>,
}
pub struct DecodeQueries(pub HashMap<String, Box<RawValue>>);
#[axum::async_trait]
impl<S> FromRequestParts<S> for DecodeQueries
where
S: Send + Sync,
{
type Rejection = Response;
async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result<Self, Self::Rejection> {
Ok(DecodeQueries::from_uri(&parts.uri).unwrap_or_else(|| DecodeQueries(HashMap::new())))
}
}
impl DecodeQueries {
fn from_uri(uri: &Uri) -> Option<Self> {
let query = uri.query();
if query.is_none() {
return None;
}
let query = query.unwrap();
let include_query = serde_urlencoded::from_str::<IncludeQuery>(query)
.map(|x| x.include_query)
.ok()
.flatten()
.unwrap_or_default();
let parse_query_args = include_query
.split(",")
.map(|s| s.to_string())
.collect::<Vec<_>>();
let mut args = HashMap::new();
if !parse_query_args.is_empty() {
let queries =
serde_urlencoded::from_str::<HashMap<String, String>>(query).unwrap_or_default();
parse_query_args.iter().for_each(|h| {
if let Some(v) = queries.get(h) {
args.insert(h.to_string(), to_raw_value(v));
}
});
}
Some(DecodeQueries(args))
}
}
// impl<'c> PushArgs<'c> {
// pub fn insert<K: Into<String>, V: Into<Box<RawValue>>>(&mut self, k: K, v: V) {
// self.extra.insert(k.into(), v.into());
// }
// }
#[derive(Deserialize)]
pub struct RequestQuery {
pub raw: Option<bool>,
pub wrap_body: Option<bool>,
pub include_header: Option<String>,
}
fn restructure_cloudevents_metadata(
mut p: HashMap<String, Box<RawValue>>,
) -> Result<HashMap<String, Box<RawValue>>, Error> {
let data = p
.remove("data")
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
let str = data.to_string();
let wrap_body = str.len() > 0 && str.chars().next().unwrap() != '{';
if wrap_body {
let args = serde_json::from_str::<Option<Box<RawValue>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
let mut hm = HashMap::new();
hm.insert("body".to_string(), args);
hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p));
Ok(hm)
} else {
let mut hm = serde_json::from_str::<Option<HashMap<String, Box<JsonRawValue>>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)))?
.unwrap_or_else(HashMap::new);
hm.insert("WEBHOOK__METADATA__".to_string(), to_raw_value(&p));
Ok(hm)
}
}
impl PushArgsOwned {
async fn from_json(
mut extra: HashMap<String, Box<RawValue>>,
use_raw: bool,
force_wrap_body: bool,
str: String,
) -> Result<Self, Response> {
if use_raw {
extra.insert("raw_string".to_string(), to_raw_value(&str));
}
let wrap_body = force_wrap_body || str.len() > 0 && str.chars().next().unwrap() != '{';
if wrap_body {
let args = serde_json::from_str::<Option<Box<RawValue>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())?
.unwrap_or_else(|| to_raw_value(&serde_json::Value::Null));
let mut hm = HashMap::new();
hm.insert("body".to_string(), args);
Ok(PushArgsOwned { extra: Some(extra), args: hm })
} else {
let hm = serde_json::from_str::<Option<HashMap<String, Box<JsonRawValue>>>>(&str)
.map_err(|e| Error::BadRequest(format!("invalid json: {}", e)).into_response())?
.unwrap_or_else(HashMap::new);
Ok(PushArgsOwned { extra: Some(extra), args: hm })
}
}
async fn from_ce_json(
mut extra: HashMap<String, Box<RawValue>>,
use_raw: bool,
str: String,
) -> Result<Self, Response> {
if use_raw {
extra.insert("raw_string".to_string(), to_raw_value(&str));
}
let hm = serde_json::from_str::<HashMap<String, Box<RawValue>>>(&str).map_err(|e| {
Error::BadRequest(format!("invalid cloudevents+json: {}", e)).into_response()
})?;
let hm = restructure_cloudevents_metadata(hm).map_err(|e| e.into_response())?;
Ok(PushArgsOwned { extra: Some(extra), args: hm })
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test]
async fn test_cloudevents_json_payload() {
let r1 = r#"
{
"specversion" : "1.0",
"type" : "com.example.someevent",
"source" : "/mycontext",
"subject": null,
"id" : "C234-1234-1234",
"time" : "2018-04-05T17:31:00Z",
"comexampleextension1" : "value",
"comexampleothervalue" : 5,
"datacontenttype" : "application/json",
"data" : {
"appinfoA" : "abc",
"appinfoB" : 123,
"appinfoC" : true
}
}
"#;
let r2 = r#"
{
"specversion" : "1.0",
"type" : "com.example.someevent",
"source" : "/mycontext",
"subject": null,
"id" : "C234-1234-1234",
"time" : "2018-04-05T17:31:00Z",
"comexampleextension1" : "value",
"comexampleothervalue" : 5,
"datacontenttype" : "application/json",
"data" : 1.5
}
"#;
let extra = HashMap::new();
let a1 = PushArgsOwned::from_ce_json(extra.clone(), false, r1.to_string())
.await
.expect("Failed to parse the cloudevent");
let a2 = PushArgsOwned::from_ce_json(extra.clone(), false, r2.to_string())
.await
.expect("Failed to parse the cloudevent");
a1.args.get("WEBHOOK__METADATA__").expect(
"CloudEvents should generate a neighboring `webhook-metadata` field in PushArgs",
);
assert_eq!(
a2.args
.get("body")
.expect("Cloud events with a data field with no wrapping curly brackets should be inside of a `body` field in PushArgs")
.to_string(),
"1.5"
);
}
}
#[axum::async_trait]
impl<S> FromRequest<S, axum::body::Body> for PushArgsOwned
where
S: Send + Sync,
{
type Rejection = Response;
async fn from_request(
req: Request<axum::body::Body>,
_state: &S,
) -> Result<Self, Self::Rejection> {
let (content_type, mut extra, use_raw, wrap_body) = {
let headers_map = req.headers();
let content_type_header = headers_map.get(CONTENT_TYPE);
let content_type = content_type_header.and_then(|value| value.to_str().ok());
let uri = req.uri();
let query = Query::<RequestQuery>::try_from_uri(uri).unwrap().0;
let mut extra = build_extra(&headers_map, query.include_header);
let query_decode = DecodeQueries::from_uri(uri);
if let Some(DecodeQueries(queries)) = query_decode {
extra.extend(queries);
}
let raw = query.raw.as_ref().is_some_and(|x| *x);
let wrap_body = query.wrap_body.as_ref().is_some_and(|x| *x);
(content_type, extra, raw, wrap_body)
};
let no_content_type = content_type.is_none();
if no_content_type || content_type.unwrap().starts_with("application/json") {
let bytes = Bytes::from_request(req, _state)
.await
.map_err(IntoResponse::into_response)?;
if no_content_type && bytes.is_empty() {
if use_raw {
extra.insert("raw_string".to_string(), to_raw_value(&"".to_string()));
}
let mut args = HashMap::new();
if wrap_body {
args.insert("body".to_string(), to_raw_value(&serde_json::json!({})));
}
return Ok(PushArgsOwned { extra: Some(extra), args: args });
}
let str = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
PushArgsOwned::from_json(extra, use_raw, wrap_body, str).await
} else if content_type
.unwrap()
.starts_with("application/cloudevents+json")
{
let bytes = Bytes::from_request(req, _state)
.await
.map_err(IntoResponse::into_response)?;
let str = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
PushArgsOwned::from_ce_json(extra, use_raw, str).await
} else if content_type
.unwrap()
.starts_with("application/cloudevents-batch+json")
{
Err(
Error::BadRequest(format!("Cloud events batching is not supported yet"))
.into_response(),
)
} else if content_type.unwrap().starts_with("text/plain") {
let bytes = Bytes::from_request(req, _state)
.await
.map_err(IntoResponse::into_response)?;
let str = String::from_utf8(bytes.to_vec())
.map_err(|e| Error::BadRequest(format!("invalid utf8: {}", e)).into_response())?;
extra.insert("raw_string".to_string(), to_raw_value(&str));
Ok(PushArgsOwned { extra: Some(extra), args: HashMap::new() })
} else if content_type
.unwrap()
.starts_with("application/x-www-form-urlencoded")
{
let bytes = Bytes::from_request(req, _state)
.await
.map_err(IntoResponse::into_response)?;
if use_raw {
let raw_string = String::from_utf8(bytes.to_vec()).map_err(|e| {
Error::BadRequest(format!("invalid utf8: {}", e)).into_response()
})?;
extra.insert("raw_string".to_string(), to_raw_value(&raw_string));
}
let payload: HashMap<String, Option<String>> = serde_urlencoded::from_bytes(&bytes)
.map_err(|e| {
Error::BadRequest(format!("invalid urlencoded data: {}", e)).into_response()
})?;
let payload = payload
.into_iter()
.map(|(k, v)| (k, to_raw_value(&v)))
.collect::<HashMap<_, _>>();
return Ok(PushArgsOwned { extra: Some(extra), args: payload });
} else {
Err(StatusCode::UNSUPPORTED_MEDIA_TYPE.into_response())
}
}
}
lazy_static::lazy_static! {
static ref INCLUDE_HEADERS: Vec<String> = std::env::var("INCLUDE_HEADERS")
.ok().map(|x| x
.split(',')
.map(|s| s.to_string())
.collect()).unwrap_or_default();
}
pub fn build_extra(
headers: &HeaderMap,
include_header: Option<String>,
) -> HashMap<String, Box<RawValue>> {
let mut args = HashMap::new();
let whitelist = include_header
.map(|s| s.split(",").map(|s| s.to_string()).collect::<Vec<_>>())
.unwrap_or_default();
whitelist
.iter()
.chain(INCLUDE_HEADERS.iter())
.for_each(|h| {
if let Some(v) = headers.get(h) {
args.insert(
h.to_string().to_lowercase().replace('-', "_"),
to_raw_value(&v.to_str().unwrap().to_string()),
);
}
});
args
}
impl PushArgsOwned {
pub fn empty() -> Self {
PushArgsOwned { extra: None, args: HashMap::new() }
@@ -3229,6 +2879,26 @@ pub async fn push<'c, 'd>(
None,
)
}
JobPayload::AppScript {
id, // app_script(id).
path,
language,
cache_ttl,
} => (
Some(id.0),
path,
None,
JobKind::AppScript,
None,
None,
Some(language),
None,
None,
None,
cache_ttl,
None,
None,
),
JobPayload::ScriptHub { path } => {
if path == "hub/7771/slack" || path == "hub/7836/slack" {
permissioned_as = SUPERADMIN_NOTIFICATION_EMAIL.to_string();
@@ -3989,6 +3659,7 @@ pub async fn push<'c, 'd>(
JobKind::DeploymentCallback => "jobs.run.deployment_callback",
JobKind::FlowScript => "jobs.run.flow_script",
JobKind::FlowNode => "jobs.run.flow_node",
JobKind::AppScript => "jobs.run.app_script",
};
let audit_author = if format!("u/{user}") != permissioned_as && user != permissioned_as {
@@ -4018,7 +3689,6 @@ pub async fn push<'c, 'd>(
script_path.as_ref().map(|x| x.as_str()),
Some(hm),
)
.instrument(tracing::info_span!("job_run", email = &email))
.await?;
}
backend/windmill-worker/Cargo.toml
+15 -3
View File
@@ -18,7 +18,12 @@ parquet = ["windmill-common/parquet", "dep:object_store"]
flow_testing = []
cloud = []
sqlx = []
deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core", "dep:deno_ast", "dep:deno_tls"]
deno_core = ["dep:deno_fetch", "dep:deno_webidl", "dep:deno_web", "dep:deno_net", "dep:deno_console", "dep:deno_url", "dep:deno_core",
"dep:deno_ast", "dep:deno_tls", "dep:deno_permissions"]
otel = ["windmill-common/otel", "dep:opentelemetry"]
dind = ["dep:bollard"]
php = ["dep:windmill-parser-php"]
mysql = ["dep:mysql_async"]
[dependencies]
windmill-queue.workspace = true
@@ -35,7 +40,7 @@ windmill-parser-py-imports.workspace = true
windmill-parser-bash.workspace = true
windmill-parser-sql.workspace = true
windmill-parser-graphql.workspace = true
windmill-parser-php.workspace = true
windmill-parser-php = { workspace = true, optional = true }
windmill-git-sync.workspace = true
sqlx.workspace = true
uuid.workspace = true
@@ -69,9 +74,11 @@ deno_url = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
deno_ast = { workspace = true, optional = true }
deno_tls = { workspace = true, optional = true }
deno_permissions = { workspace = true, optional = true }
postgres-native-tls.workspace = true
native-tls.workspace = true
mysql_async.workspace = true
mysql_async = { workspace = true, optional = true }
base64.workspace = true
gcp_auth = { workspace = true, optional = true }
rust_decimal.workspace = true
@@ -93,6 +100,9 @@ yaml-rust.workspace = true
swc_ecma_parser.workspace = true
backon.workspace = true
opentelemetry = { workspace = true, optional = true }
bollard = { workspace = true, optional = true }
[build-dependencies]
deno_fetch = { workspace = true, optional = true }
deno_webidl = { workspace = true, optional = true }
@@ -103,4 +113,6 @@ deno_url = { workspace = true, optional = true }
deno_core = { workspace = true, optional = true }
deno_ast = { workspace = true, optional = true }
deno_tls = { workspace = true, optional = true }
deno_permissions = { workspace = true, optional = true }
zstd.workspace = true
backend/windmill-worker/build.rs
+8 -7
View File
@@ -25,7 +25,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
_url: &deno_core::url::Url,
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -34,7 +34,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
_p: &'a std::path::Path,
_api_name: &str,
) -> Result<Cow<'a, Path>, deno_core::error::AnyError> {
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
}
@@ -53,7 +53,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -61,7 +61,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -69,7 +69,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_host: &(T, Option<u16>),
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
unreachable!("snapshotting")
}
@@ -77,7 +77,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_: &'a Path,
_: &str,
) -> Result<Cow<'a, Path>, deno_core::anyhow::Error> {
) -> Result<Cow<'a, Path>, deno_permissions::PermissionCheckError> {
todo!()
}
}
@@ -95,6 +95,7 @@ fn main() {
println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
let exts = vec![
// deno_telemetry::deno_telemetry::init_ops_and_esm(),
deno_webidl::deno_webidl::init_ops_and_esm(),
deno_url::deno_url::init_ops_and_esm(),
deno_console::deno_console::init_ops_and_esm(),
@@ -116,10 +117,10 @@ fn main() {
deno_core::snapshot::CreateSnapshotOptions {
cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
startup_snapshot: None,
extension_transpiler: None,
extensions: exts,
with_runtime_cb: None,
skip_op_registration: false,
extension_transpiler: None,
},
None,
)
backend/windmill-worker/src/ansible_executor.rs
+2 -2
View File
@@ -115,8 +115,8 @@ async fn handle_ansible_python_deps(
job_dir,
worker_dir,
&mut Some(occupancy_metrics),
true,
true,
false,
false,
)
.await?;
additional_python_paths.append(&mut venv_path);
backend/windmill-worker/src/bash_executor.rs
+227 -1
View File
@@ -1,14 +1,30 @@
use std::{collections::HashMap, fs, process::Stdio};
#[cfg(feature = "dind")]
use bollard::container::{
KillContainerOptions, RemoveContainerOptions, StatsOptions, StopContainerOptions,
};
#[cfg(feature = "dind")]
use futures::{stream, StreamExt, TryStreamExt};
use regex::Regex;
use serde_json::{json, value::RawValue};
use sqlx::types::Json;
use tokio::process::Command;
#[cfg(feature = "dind")]
use uuid::Uuid;
use windmill_common::{
error::Error,
jobs::QueuedJob,
worker::{to_raw_value, write_file},
};
#[cfg(feature = "dind")]
use windmill_common::DB;
#[cfg(feature = "dind")]
use windmill_common::error::to_anyhow;
use windmill_queue::{append_logs, CanceledBy};
lazy_static::lazy_static! {
@@ -22,6 +38,9 @@ lazy_static::lazy_static! {
static ref RE_POWERSHELL_IMPORTS: Regex = Regex::new(r#"^Import-Module\s+(?:-Name\s+)?"?([^-\s"]+)"?"#).unwrap();
}
#[cfg(feature = "dind")]
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::{
common::{
build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process,
@@ -54,8 +73,14 @@ pub async fn handle_bash_job(
worker_name: &str,
envs: HashMap<String, String>,
occupancy_metrics: &mut OccupancyMetrics,
_killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
) -> Result<Box<RawValue>, Error> {
let logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string();
let annotation = windmill_common::worker::BashAnnotations::parse(&content);
let mut logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string();
if annotation.docker {
logs1.push_str("docker mode\n");
}
append_logs(&job.id, &job.workspace_id, logs1, db).await;
write_file(job_dir, "main.sh", &format!("set -e\n{content}"))?;
@@ -193,6 +218,22 @@ exit $exit_status
)
.await?;
#[cfg(feature = "dind")]
if annotation.docker {
return handle_docker_job(
job.id,
&job.workspace_id,
db,
job.timeout,
mem_peak,
canceled_by,
worker_name,
occupancy_metrics,
_killpill_rx,
)
.await;
}
let result_json_path = format!("{job_dir}/result.json");
if let Ok(metadata) = tokio::fs::metadata(&result_json_path).await {
if metadata.len() > 0 {
@@ -214,6 +255,7 @@ exit $exit_status
.await?
.trim()
.to_string();
return Ok(to_raw_value(&json!(result)));
}
@@ -222,6 +264,190 @@ exit $exit_status
)))
}
#[cfg(feature = "dind")]
async fn handle_docker_job(
job_id: Uuid,
workspace_id: &str,
db: &DB,
job_timeout: Option<i32>,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
occupancy_metrics: &mut OccupancyMetrics,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
) -> Result<Box<RawValue>, Error> {
let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow)?;
let container_id = job_id.to_string();
let inspected = client.inspect_container(&container_id, None).await;
if inspected.is_err() || inspected.unwrap().state.is_none() {
return Ok(to_raw_value(&format!(
"Container not found at {job_id}, you must use --name and not --rm it"
)));
}
let wait_f = async {
let wait = client
.wait_container::<String>(&container_id, None)
.try_collect::<Vec<_>>()
.await
.map_err(|e| {
tracing::error!("Error waiting for container: {:?}", e);
anyhow::anyhow!("Error waiting for container")
})?;
let waited = wait.first().map(|x| x.status_code);
Ok(waited)
};
let ncontainer_id = container_id.to_string();
let w_id = workspace_id.to_string();
let j_id = job_id.clone();
let db2 = db.clone();
let (tx, mut rx) = tokio::sync::broadcast::channel::<()>(1);
let mut killpill_rx = killpill_rx.resubscribe();
let logs = tokio::spawn(async move {
let client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow);
if let Ok(client) = client {
let mut log_stream = client.logs(
&ncontainer_id,
Some(bollard::container::LogsOptions {
follow: true,
stdout: true,
stderr: true,
tail: "all",
..Default::default()
}),
);
loop {
tokio::select! {
log = log_stream.next() => {
match log {
Some(Ok(log)) => {
append_logs(&j_id, w_id.clone(), log.to_string(), db2.clone()).await;
}
Some(Err(e)) => {
tracing::error!("Error getting logs: {:?}", e);
}
_ => {
tracing::error!("End of stream");
return
}
};
},
_ = killpill_rx.recv() => {
tracing::error!("killing container after receving killpill");
if let Err(e) = client
.stop_container(&ncontainer_id, Some(StopContainerOptions { t: 3 }))
.await
{
tracing::error!("Error stopping container: {:?}", e);
}
return
},
_ = rx.recv() => {
return
}
}
}
}
});
let mem_client = bollard::Docker::connect_with_unix_defaults().map_err(to_anyhow);
let ncontainer_id = container_id.clone();
let result = run_future_with_polling_update_job_poller(
job_id,
job_timeout,
db,
mem_peak,
canceled_by,
wait_f,
worker_name,
workspace_id,
&mut Some(occupancy_metrics),
Box::pin(match mem_client {
Ok(client) => client
.stats(
&ncontainer_id,
Some(StatsOptions { stream: true, one_shot: false }),
)
.map(|x| {
x.map(|x| x.memory_stats.usage.map(|m| m / 1024).unwrap_or_default() as i32)
.unwrap_or_default()
})
.boxed(),
_ => stream::once(async { 0 }).boxed(),
}),
)
.await;
if let Err(e) = result {
if !logs.is_finished() {
let _ = tx.send(());
let _ = logs.await;
}
if container_is_alive(&client, &container_id).await {
kill_container(&client, &container_id, "SIGINT").await;
if container_is_alive(&client, &container_id).await {
tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
if let Err(e) = client
.stop_container(&container_id, Some(StopContainerOptions { t: 3 }))
.await
{
tracing::error!("Error stopping container: {:?}", e);
}
}
}
return Err(e);
}
if let Err(e) = client
.remove_container(
&container_id,
Some(RemoveContainerOptions { force: true, ..Default::default() }),
)
.await
{
tracing::error!("Error removing container: {:?}", e);
}
let result = result.unwrap();
return Ok(to_raw_value(&json!(format!(
"Docker exit status: {}",
result
.map(|x| x.to_string())
.unwrap_or_else(|| "none".to_string())
))));
}
#[cfg(feature = "dind")]
async fn kill_container(client: &bollard::Docker, container_id: &str, signal: &str) {
if let Err(e) = client
.kill_container(&container_id, Some(KillContainerOptions { signal }))
.await
{
tracing::error!("Error killing container with signal {signal}: {:?}", e);
}
}
#[cfg(feature = "dind")]
async fn container_is_alive(client: &bollard::Docker, container_id: &str) -> bool {
let inspect = client.inspect_container(container_id, None).await;
if let Ok(inspect) = inspect {
let r = inspect
.state
.map(|x| x.running.unwrap_or_default())
.unwrap_or_default();
tracing::error!("Container {container_id} is alive: {r}");
r
} else {
false
}
}
fn raw_to_string(x: &str) -> String {
match serde_json::from_str::<serde_json::Value>(x) {
Ok(serde_json::Value::String(x)) => x,
backend/windmill-worker/src/bigquery_executor.rs
+1
View File
@@ -366,6 +366,7 @@ pub async fn do_bigquery(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/common.rs
+122 -71
View File
@@ -1,11 +1,12 @@
use async_recursion::async_recursion;
use itertools::Itertools;
use lazy_static::lazy_static;
use regex::Regex;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
use serde_json::{json, Value};
use sha2::Digest;
use sqlx::types::Json;
use sqlx::{Pool, Postgres};
use tokio::process::Command;
@@ -21,8 +22,10 @@ use windmill_common::worker::{
to_raw_value, write_file, CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG,
};
use windmill_common::{
cache::Cache,
error::{self, Error},
jobs::QueuedJob,
scripts::ScriptHash,
variables::ContextualVariable,
};
@@ -32,6 +35,7 @@ use std::path::Path;
use std::{
collections::{hash_map::DefaultHasher, HashMap},
hash::{Hash, Hasher},
sync::Arc,
time::Duration,
};
@@ -633,40 +637,59 @@ pub async fn resolve_job_timeout(
}
}
pub async fn hash_args(
async fn hash_args(
_db: &DB,
_client: &AuthedClient,
_workspace_id: &str,
_job_id: &Uuid,
v: &Option<sqlx::types::Json<HashMap<String, Box<RawValue>>>>,
) -> String {
if let Some(vs) = v {
let mut dh = DefaultHasher::new();
let hm = &vs.0;
v: &Option<Json<HashMap<String, Box<RawValue>>>>,
hasher: &mut sha2::Sha256,
) {
if let Some(Json(hm)) = v {
for k in hm.keys().sorted() {
k.hash(&mut dh);
hasher.update(k.as_bytes());
let arg_value = hm.get(k).unwrap();
#[cfg(feature = "parquet")]
let (_, arg_additions) =
arg_value_hash_additions(_db, _client, _workspace_id, hm.get(k).unwrap()).await;
arg_value.get().hash(&mut dh);
hasher.update(arg_value.get().as_bytes());
#[cfg(feature = "parquet")]
for (_, arg_addition) in arg_additions {
arg_addition.hash(&mut dh);
hasher.update(arg_addition.as_bytes());
}
}
hex::encode(dh.finish().to_be_bytes())
} else {
"empty_args".to_string()
}
}
pub async fn cached_result_path(
db: &DB,
client: &AuthedClient,
job: &QueuedJob,
raw_code: Option<&String>,
raw_lock: Option<&String>,
raw_flow: Option<&Json<Box<RawValue>>>,
) -> String {
let mut hasher = sha2::Sha256::new();
hasher.update(&[job.job_kind as u8]);
if let Some(ScriptHash(hash)) = job.script_hash {
hasher.update(&hash.to_le_bytes())
} else if let None = job.script_hash {
job.script_path
.as_ref()
.inspect(|x| hasher.update(x.as_bytes()));
raw_code.inspect(|x| hasher.update(x));
raw_lock.inspect(|x| hasher.update(x));
raw_flow.inspect(|x| hasher.update(x.get()));
}
hash_args(db, client, &job.workspace_id, &job.args, &mut hasher).await;
format!("g/results/{:032x}", hasher.finalize())
}
#[cfg(feature = "parquet")]
async fn get_workspace_s3_resource_path(
db: &DB,
client: &AuthedClient,
workspace_id: &str,
storage: Option<String>,
storage: Option<&String>,
) -> windmill_common::error::Result<Option<ObjectStoreResource>> {
use windmill_common::{
job_s3_helpers_ee::get_s3_resource_internal, s3_helpers::StorageResourceType,
@@ -750,7 +773,7 @@ async fn arg_value_hash_additions(
let mut storage = None;
if let Ok(s3_object) = parsed_value {
let s3_resource_opt =
get_workspace_s3_resource_path(db, client, workspace_id, s3_object.storage.clone())
get_workspace_s3_resource_path(db, client, workspace_id, s3_object.storage.as_ref())
.await;
storage = s3_object.storage.clone();
@@ -769,68 +792,93 @@ struct CachedResource {
expire: i64,
#[serde(skip_serializing_if = "Option::is_none")]
s3_etags: Option<HashMap<String, String>>,
value: Box<RawValue>,
value: Arc<Box<RawValue>>,
storage: Option<String>,
}
impl CachedResource {
fn expired(&self) -> bool {
self.expire <= chrono::Utc::now().timestamp()
}
}
lazy_static! {
/// In-memory cache for resources.
static ref CACHED_RESULTS: Cache<String, Arc<CachedResource>> = Cache::new(1000);
}
pub async fn get_cached_resource_value_if_valid(
_db: &DB,
client: &AuthedClient,
_job_id: &Uuid,
_workspace_id: &str,
cached_res_path: &str,
) -> Option<Box<RawValue>> {
let resource_opt = client
.get_resource_value::<CachedResource>(cached_res_path)
) -> Option<Arc<Box<RawValue>>> {
let resource = match CACHED_RESULTS
.get_value_or_guard_async(cached_res_path)
.await
.ok();
if let Some(cached_resource) = resource_opt {
if cached_resource.expire <= chrono::Utc::now().timestamp() {
// cache expired
{
// check for cache expiration.
Ok(resource) if resource.expired() => {
let _ = CACHED_RESULTS.remove(cached_res_path);
return None;
}
#[cfg(feature = "parquet")]
// resource is in cache and not expired, return it.
Ok(resource) => resource,
// resource is not in cache, fetch it from the database.
Err(entry) => match client
.get_resource_value::<CachedResource>(cached_res_path)
.await
.ok()
{
let s3_etags = cached_resource.s3_etags.unwrap_or_default();
let object_store_resource_opt: Option<ObjectStoreResource> = if s3_etags.is_empty() {
None
} else {
get_workspace_s3_resource_path(
_db,
&client,
_workspace_id,
cached_resource.storage.clone(),
)
None => return None,
// check for cache expiration.
Some(resource) if resource.expired() => return None,
Some(resource) => {
let resource = Arc::new(resource);
let _ = entry.insert(resource.clone());
resource
}
},
};
#[cfg(feature = "parquet")]
{
let empty_etags = HashMap::new();
let s3_etags = resource.s3_etags.as_ref().unwrap_or(&empty_etags);
let object_store_resource_opt: Option<ObjectStoreResource> = if s3_etags.is_empty() {
None
} else {
get_workspace_s3_resource_path(_db, &client, _workspace_id, resource.storage.as_ref())
.await
.ok()
.flatten()
};
};
if !s3_etags.is_empty() && object_store_resource_opt.is_none() {
tracing::warn!("Cached result references s3 files that are not retrievable anymore because the workspace S3 resource can't be fetched. Cache will be invalidated");
return None;
}
for (s3_file_key, s3_file_etag) in s3_etags {
if let Some(mut object_store_resource) = object_store_resource_opt.clone() {
let etag = get_etag_or_empty(
&mut object_store_resource,
S3Object {
s3: s3_file_key.clone(),
storage: cached_resource.storage.clone(),
filename: None,
},
)
.await;
if etag.is_none() || etag.clone().unwrap() != s3_file_etag {
tracing::warn!("S3 file etag for '{}' has changed. Value from cache is {:?} while current value from S3 is {:?}. Cache will be invalidated", s3_file_key.clone(), s3_file_etag, etag);
return None;
}
if !s3_etags.is_empty() && object_store_resource_opt.is_none() {
tracing::warn!("Cached result references s3 files that are not retrievable anymore because the workspace S3 resource can't be fetched. Cache will be invalidated");
return None;
}
for (s3_file_key, s3_file_etag) in s3_etags {
if let Some(object_store_resource) = object_store_resource_opt.as_ref() {
let etag = get_etag_or_empty(
object_store_resource,
S3Object {
s3: s3_file_key.clone(),
storage: resource.storage.clone(),
filename: None,
},
)
.await;
if etag.as_ref() != Some(s3_file_etag) {
tracing::warn!("S3 file etag for '{}' has changed. Value from cache is {:?} while current value from S3 is {:?}. Cache will be invalidated", s3_file_key.clone(), s3_file_etag, etag);
return None;
}
}
}
return Some(cached_resource.value);
}
return None;
Some(resource.value.clone())
}
pub async fn save_in_cache(
@@ -838,13 +886,13 @@ pub async fn save_in_cache(
_client: &AuthedClient,
job: &QueuedJob,
cached_path: String,
r: &Box<RawValue>,
r: Arc<Box<RawValue>>,
) {
let expire = chrono::Utc::now().timestamp() + job.cache_ttl.unwrap() as i64;
#[cfg(feature = "parquet")]
let (storage, s3_etags) =
arg_value_hash_additions(db, _client, job.workspace_id.as_str(), r).await;
arg_value_hash_additions(db, _client, job.workspace_id.as_str(), &r).await;
#[cfg(feature = "parquet")]
let s3_etags = if s3_etags.is_empty() {
@@ -856,17 +904,17 @@ pub async fn save_in_cache(
#[cfg(not(feature = "parquet"))]
let (storage, s3_etags) = (None, None);
let store_cache_resource = CachedResource { expire, s3_etags, value: r.clone(), storage };
let raw_json = sqlx::types::Json(store_cache_resource);
let store_cache_resource = CachedResource { expire, s3_etags, value: r, storage };
let raw_json = Json(&store_cache_resource);
if let Err(e) = sqlx::query!(
"INSERT INTO resource
(workspace_id, path, value, resource_type, created_by, edited_at)
VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path)
DO UPDATE SET value = $3, edited_at = now()",
(workspace_id, path, value, resource_type, created_by, edited_at)
VALUES ($1, $2, $3, $4, $5, now()) ON CONFLICT (workspace_id, path)
DO UPDATE SET value = $3, edited_at = now()",
job.workspace_id,
cached_path,
raw_json as sqlx::types::Json<CachedResource>,
&cached_path,
raw_json as Json<&CachedResource>,
"cache",
job.created_by
)
@@ -875,6 +923,9 @@ pub async fn save_in_cache(
{
tracing::error!("Error creating cache resource {e:#}")
}
// Cache result in-memory.
CACHED_RESULTS.insert(cached_path, Arc::new(store_cache_resource));
}
fn tentatively_improve_error(err: Error, executable: &str) -> Error {
@@ -884,11 +935,11 @@ fn tentatively_improve_error(err: Error, executable: &str) -> Error {
#[cfg(windows)]
let err_msg = "program not found";
if err
.to_string()
.contains(&err_msg)
{
return Error::InternalErr(format!("Executable {executable} not found on worker. PATH: {}", *PATH_ENV));
if err.to_string().contains(&err_msg) {
return Error::InternalErr(format!(
"Executable {executable} not found on worker. PATH: {}",
*PATH_ENV
));
}
return err;
}
backend/windmill-worker/src/dedicated_worker.rs
+7 -7
View File
@@ -16,8 +16,7 @@ use windmill_common::error::Error;
use windmill_common::flows::FlowValue;
use windmill_common::worker::WORKER_CONFIG;
use windmill_common::{
cache,
error,
cache, error,
flows::{FlowModule, FlowModuleValue},
jobs::QueuedJob,
scripts::{ScriptHash, ScriptLang},
@@ -416,17 +415,18 @@ async fn spawn_dedicated_workers_for_flow(
job_completed_tx,
Some(module.id.clone()),
)
.await
.await
{
workers.push(dedi_w);
}
},
}
Err(err) => tracing::error!(
"failed to get script for module: {:?}, err: {:?}",
module, err
)
module,
err
),
}
},
}
FlowModuleValue::Flow { .. } => (),
FlowModuleValue::Identity => (),
}
backend/windmill-worker/src/graphql_executor.rs
+2 -1
View File
@@ -1,7 +1,7 @@
use std::collections::HashMap;
use anyhow::anyhow;
use futures::TryStreamExt;
use futures::{stream, TryStreamExt};
use serde_json::{json, value::RawValue};
use sqlx::types::Json;
use windmill_common::jobs::QueuedJob;
@@ -154,6 +154,7 @@ pub async fn do_graphql(
worker_name,
&job.workspace_id,
&mut Some(occupation_metrics),
Box::pin(stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/handle_child.rs
+20 -18
View File
@@ -87,7 +87,7 @@ async fn kill_process_tree(pid: Option<u32>) -> Result<(), String> {
/// - update the `last_line` and `logs` strings with the program output
/// - update "queue"."last_ping" every five seconds
/// - kill process if we exceed timeout or "queue"."canceled" is set
#[tracing::instrument(level = "trace", skip_all)]
#[tracing::instrument(name="run_subprocess", level = "info", skip_all, fields(otel.name = %child_name))]
pub async fn handle_child(
job_id: &Uuid,
db: &Pool<Postgres>,
@@ -126,7 +126,7 @@ pub async fn handle_child(
let (tx, rx) = broadcast::channel::<()>(3);
let mut rx2 = tx.subscribe();
let output = child_joined_output_stream(&mut child);
let output = child_joined_output_stream(&mut child, job_id.clone());
let job_id = job_id.clone();
@@ -137,7 +137,9 @@ pub async fn handle_child(
db,
mem_peak,
canceled_by_ref,
|| get_mem_peak(pid, nsjail),
Box::pin(stream::unfold((), move |_| async move {
Some((get_mem_peak(pid, nsjail).await, ()))
})),
worker,
w_id,
rx,
@@ -435,8 +437,6 @@ pub async fn handle_child(
}
}
async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
if pid.is_none() {
return -1;
@@ -488,7 +488,7 @@ async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
}
}
pub async fn run_future_with_polling_update_job_poller<Fut, T>(
pub async fn run_future_with_polling_update_job_poller<Fut, T, S>(
job_id: Uuid,
timeout: Option<i32>,
db: &DB,
@@ -498,9 +498,11 @@ pub async fn run_future_with_polling_update_job_poller<Fut, T>(
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
get_mem: S,
) -> error::Result<T>
where
Fut: Future<Output = anyhow::Result<T>>,
S: stream::Stream<Item = i32> + Unpin,
{
let (tx, rx) = broadcast::channel::<()>(3);
@@ -509,7 +511,7 @@ where
db,
mem_peak,
canceled_by_ref,
|| async { 0 },
get_mem,
worker_name,
w_id,
rx,
@@ -550,20 +552,19 @@ pub enum UpdateJobPollingExit {
AlreadyCompleted,
}
pub async fn update_job_poller<F, Fut>(
pub async fn update_job_poller<S>(
job_id: Uuid,
db: &DB,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
get_mem: F,
mut get_mem: S,
worker_name: &str,
w_id: &str,
mut rx: broadcast::Receiver<()>,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> UpdateJobPollingExit
where
F: Fn() -> Fut,
Fut: Future<Output = i32>,
S: stream::Stream<Item = i32> + Unpin,
{
let update_job_interval = Duration::from_millis(500);
@@ -608,7 +609,7 @@ where
.expect("update worker ping");
}
}
let current_mem = get_mem().await;
let current_mem = get_mem.next().await.unwrap_or(0);
if current_mem > *mem_peak {
*mem_peak = current_mem
}
@@ -679,6 +680,7 @@ where
/// builds a stream joining both stdout and stderr each read line by line
fn child_joined_output_stream(
child: &mut Child,
job_id: Uuid,
) -> impl stream::FusedStream<Item = io::Result<String>> {
let stderr = child
.stderr
@@ -692,24 +694,24 @@ fn child_joined_output_stream(
let stdout = BufReader::new(stdout).lines();
let stderr = BufReader::new(stderr).lines();
stream::select(lines_to_stream(stderr, true), lines_to_stream(stdout, false))
stream::select(
lines_to_stream(stderr, true, job_id.clone()),
lines_to_stream(stdout, false, job_id),
)
}
pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
mut lines: tokio::io::Lines<R>,
stderr: bool,
job_id: Uuid,
) -> impl futures::Stream<Item = io::Result<String>> {
stream::poll_fn(move |cx| {
std::pin::Pin::new(&mut lines)
.poll_next_line(cx)
.map(|result| {
process_streaming_log_lines(result, stderr)
})
.map(|result| process_streaming_log_lines(result, stderr, &job_id))
})
}
pub fn process_status(status: ExitStatus) -> error::Result<()> {
if status.success() {
Ok(())
backend/windmill-worker/src/job_logger.rs
+1 -4
View File
@@ -15,7 +15,6 @@ use crate::job_logger_ee::default_disk_log_storage;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
use crate::job_logger_ee::s3_storage;
pub enum CompactLogs {
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
NotEE,
@@ -25,8 +24,6 @@ pub enum CompactLogs {
S3,
}
pub(crate) async fn append_job_logs(
job_id: Uuid,
w_id: String,
@@ -70,7 +67,7 @@ pub fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
if *NO_LOGS_AT_ALL {
return;
}
let src_str;
let src = {
src_str = RE_00.replace_all(src, "");
backend/windmill-worker/src/job_logger_ee.rs
+4 -3
View File
@@ -12,11 +12,11 @@ pub(crate) async fn s3_storage(
_job_id: Uuid,
_w_id: &String,
_db: &sqlx::Pool<sqlx::Postgres>,
_logs: &String,
_total_size: &Arc<AtomicU32>,
_logs: String,
_total_size: Arc<AtomicU32>,
_worker_name: &String,
) {
tracing::info!("Logs length of {job_id} has exceeded a threshold. Implementation to store excess on s3 in not OSS");
tracing::info!("Logs length of {_job_id} has exceeded a threshold. Implementation to store excess on s3 in not OSS");
}
pub(crate) async fn default_disk_log_storage(
@@ -34,6 +34,7 @@ pub(crate) async fn default_disk_log_storage(
pub(crate) fn process_streaming_log_lines(
r: Result<Option<String>, io::Error>,
_stderr: bool,
_job_id: &Uuid,
) -> Option<Result<String, io::Error>> {
r.transpose()
}
backend/windmill-worker/src/js_eval.rs
+40 -51
View File
@@ -7,14 +7,7 @@
*/
#[cfg(feature = "deno_core")]
use std::{
borrow::Cow,
cell::RefCell,
env,
io::{self, BufReader},
path::PathBuf,
rc::Rc,
};
use std::{borrow::Cow, cell::RefCell, env, path::PathBuf, rc::Rc};
use std::{collections::HashMap, sync::Arc};
@@ -22,7 +15,6 @@ use std::{collections::HashMap, sync::Arc};
use deno_ast::ParseParams;
#[cfg(feature = "deno_core")]
use deno_core::{
error::AnyError,
op2, serde_v8, url,
v8::{self, IsolateHandle},
Extension, JsRuntime, OpState, PollEventLoopOptions, RuntimeOptions,
@@ -31,8 +23,7 @@ use deno_core::{
use deno_fetch::FetchPermissions;
#[cfg(feature = "deno_core")]
use deno_net::NetPermissions;
#[cfg(feature = "deno_core")]
use deno_tls::{rustls::RootCertStore, rustls_pemfile};
#[cfg(feature = "deno_core")]
use deno_web::{BlobStore, TimersPermission};
#[cfg(feature = "deno_core")]
@@ -68,35 +59,35 @@ pub struct IdContext {
pub previous_id: String,
}
#[cfg(feature = "deno_core")]
pub struct ContainerRootCertStoreProvider {
root_cert_store: RootCertStore,
}
// #[cfg(feature = "deno_core")]
// pub struct ContainerRootCertStoreProvider {
// root_cert_store: RootCertStore,
// }
#[cfg(feature = "deno_core")]
impl ContainerRootCertStoreProvider {
fn new() -> ContainerRootCertStoreProvider {
return ContainerRootCertStoreProvider {
root_cert_store: deno_tls::create_default_root_cert_store(),
};
}
// #[cfg(feature = "deno_core")]
// impl ContainerRootCertStoreProvider {
// fn new() -> ContainerRootCertStoreProvider {
// return ContainerRootCertStoreProvider {
// root_cert_store: deno_tls::create_default_root_cert_store(),
// };
// }
fn add_certificate(&mut self, cert_path: String) -> io::Result<()> {
let cert_file = std::fs::File::open(cert_path)?;
let mut reader = BufReader::new(cert_file);
let pem_file = rustls_pemfile::certs(&mut reader).collect::<Result<Vec<_>, _>>()?;
// fn add_certificate(&mut self, cert_path: String) -> io::Result<()> {
// let cert_file = std::fs::File::open(cert_path)?;
// let mut reader = BufReader::new(cert_file);
// let pem_file = rustls_pemfile::certs(&mut reader).collect::<Result<Vec<_>, _>>()?;
self.root_cert_store.add_parsable_certificates(pem_file);
Ok(())
}
}
// self.root_cert_store.add_parsable_certificates(pem_file);
// Ok(())
// }
// }
#[cfg(feature = "deno_core")]
impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider {
fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> {
Ok(&self.root_cert_store)
}
}
// #[cfg(feature = "deno_core")]
// impl deno_tls::RootCertStoreProvider for ContainerRootCertStoreProvider {
// fn get_or_try_init(&self) -> Result<&RootCertStore, AnyError> {
// Ok(&self.root_cert_store)
// }
// }
#[cfg(feature = "deno_core")]
pub struct PermissionsContainer;
@@ -108,7 +99,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
_url: &deno_core::url::Url,
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
Ok(())
}
@@ -117,7 +108,7 @@ impl FetchPermissions for PermissionsContainer {
&mut self,
p: &'a std::path::Path,
_api_name: &str,
) -> Result<Cow<'a, std::path::Path>, anyhow::Error> {
) -> Result<Cow<'a, std::path::Path>, deno_permissions::PermissionCheckError> {
Ok(Cow::Borrowed(p))
}
}
@@ -136,7 +127,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
Ok(PathBuf::from(p))
}
@@ -144,7 +135,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
p: &'a str,
_api_name: &str,
) -> Result<PathBuf, deno_core::error::AnyError> {
) -> Result<PathBuf, deno_permissions::PermissionCheckError> {
Ok(PathBuf::from(p))
}
@@ -152,7 +143,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
_host: &(T, Option<u16>),
_api_name: &str,
) -> Result<(), deno_core::error::AnyError> {
) -> Result<(), deno_permissions::PermissionCheckError> {
Ok(())
}
@@ -160,7 +151,7 @@ impl NetPermissions for PermissionsContainer {
&mut self,
p: &'a std::path::Path,
_api_name: &str,
) -> Result<std::borrow::Cow<'a, std::path::Path>, AnyError> {
) -> Result<std::borrow::Cow<'a, std::path::Path>, deno_permissions::PermissionCheckError> {
Ok(Cow::Borrowed(p))
}
}
@@ -669,9 +660,12 @@ pub fn transpile_ts(expr: String) -> anyhow::Result<String> {
text: deno_core::ModuleCodeString::from(expr).into(),
})?;
Ok(parsed
.transpile(&Default::default(), &Default::default())?
.transpile(
&Default::default(),
&Default::default(),
&Default::default(),
)?
.into_source()
.into_string()?
.text)
}
@@ -814,13 +808,7 @@ pub async fn eval_fetch_timeout(
let ext = Extension { name: "windmill", ops: ops.into(), ..Default::default() };
let fetch_options = deno_fetch::Options {
root_cert_store_provider: if let Some(cert_path) = env::var("DENO_CERT").ok() {
let mut cert_store_provider = ContainerRootCertStoreProvider::new();
cert_store_provider.add_certificate(cert_path)?;
Some(Arc::new(cert_store_provider))
} else {
None
},
root_cert_store_provider: None,
user_agent: ann.useragent.unwrap_or_else(|| "windmill/beta".to_string()),
proxy: ann.proxy.map(|x| deno_tls::Proxy {
url: x.0,
@@ -930,6 +918,7 @@ pub async fn eval_fetch_timeout(
worker_name,
w_id,
&mut Some(occupation_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await
.map_err(|e| {
backend/windmill-worker/src/lib.rs
+1
View File
@@ -20,6 +20,7 @@ mod graphql_executor;
mod handle_child;
mod job_logger;
mod js_eval;
#[cfg(feature = "mysql")]
mod mysql_executor;
mod pg_executor;
mod php_executor;
backend/windmill-worker/src/mssql_executor.rs
+1
View File
@@ -160,6 +160,7 @@ pub async fn do_mssql(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/mysql_executor.rs
+1
View File
@@ -298,6 +298,7 @@ pub async fn do_mysql(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/pg_executor.rs
+1
View File
@@ -355,6 +355,7 @@ pub async fn do_postgresql(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
backend/windmill-worker/src/php_executor.rs
+4
View File
@@ -192,9 +192,13 @@ pub async fn handle_php_job(
let main_override = get_main_override(job.args.as_ref());
let write_wrapper_f = async {
#[cfg(feature = "php")]
let args =
windmill_parser_php::parse_php_signature(inner_content, main_override.clone())?.args;
#[cfg(not(feature = "php"))]
let args: Vec<windmill_parser::Arg> = vec![];
let args_to_include = args
.iter()
.filter(|x| {
backend/windmill-worker/src/python_executor.rs
+84 -59
View File
@@ -3,7 +3,7 @@ use std::{
fs,
path::Path,
process::Stdio,
sync::Arc
sync::Arc,
};
use anyhow::anyhow;
@@ -51,11 +51,11 @@ lazy_static::lazy_static! {
static ref PIP_TRUSTED_HOST: Option<String> = std::env::var("PIP_TRUSTED_HOST").ok();
static ref PIP_INDEX_CERT: Option<String> = std::env::var("PIP_INDEX_CERT").ok();
static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE")
pub static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE")
.ok().map(|flag| flag == "true").unwrap_or(false);
/// Use pip install
static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL")
pub static ref USE_PIP_INSTALL: bool = std::env::var("USE_PIP_INSTALL")
.ok().map(|flag| flag == "true").unwrap_or(false);
@@ -318,7 +318,7 @@ pub async fn uv_pip_compile(
if let Some(cert_path) = PIP_INDEX_CERT.as_ref() {
args.extend(["--cert", cert_path]);
}
tracing::debug!("uv args: {:?}", args);
tracing::error!("uv args: {:?}", args);
#[cfg(windows)]
let uv_cmd = "uv";
@@ -329,7 +329,10 @@ pub async fn uv_pip_compile(
let mut child_cmd = Command::new(uv_cmd);
child_cmd
.current_dir(job_dir)
.args(args)
.env_clear()
.env("HOME", HOME_ENV.to_string())
.env("PATH", PATH_ENV.to_string())
.args(&args)
.stdout(Stdio::piped())
.stderr(Stdio::piped());
let child_process = start_child_process(child_cmd, uv_cmd).await?;
@@ -350,7 +353,12 @@ pub async fn uv_pip_compile(
occupancy_metrics,
)
.await
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
.map_err(|e| {
Error::ExecutionErr(format!(
"Lock file generation failed.\n\ncommand: {uv_cmd} {}\n\n{e:?}",
args.join(" ")
))
})?;
}
let path_lock = format!("{job_dir}/requirements.txt");
@@ -423,12 +431,12 @@ async fn postinstall(
.ok_or(anyhow::anyhow!("Cannot convert OsString to String"))?
.to_owned();
if name == "bin" || name.contains("dist-info") {
if name == "bin" || name == "__pycache__" || name.contains("dist-info") {
continue;
}
if let Some(existing_paths) = lookup_table.get_mut(&name) {
tracing::info!(
tracing::debug!(
"Found existing package name: {:?} in {}",
entry.file_name(),
path
@@ -479,18 +487,22 @@ fn copy_dir_recursively(src: &Path, dst: &Path) -> windmill_common::error::Resul
fs::create_dir_all(dst)?;
}
tracing::debug!("Copying recursively from {:?} to {:?}", src, dst);
for entry in fs::read_dir(src)? {
let entry = entry?;
let src_path = entry.path();
let dst_path = dst.join(entry.file_name());
if src_path.is_dir() {
if src_path.is_dir() && !src_path.is_symlink() {
copy_dir_recursively(&src_path, &dst_path)?;
} else {
fs::copy(&src_path, &dst_path)?;
}
}
tracing::debug!("Finished copying recursively from {:?} to {:?}", src, dst);
Ok(())
}
@@ -529,10 +541,13 @@ pub async fn handle_python_job(
)
.await?;
tracing::debug!("Finished handling python dependencies");
if !PythonAnnotations::parse(inner_content).no_postinstall {
if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, db).await {
tracing::error!("Postinstall stage has failed. Reason: {e}");
}
tracing::debug!("Finished deps postinstall stage");
}
append_logs(
@@ -563,9 +578,12 @@ pub async fn handle_python_job(
)
.await?;
tracing::debug!("Finished preparing wrapper");
let apply_preprocessor = pre_spread.is_some();
create_args_and_out_file(&client, job, job_dir, db).await?;
tracing::debug!("Finished preparing wrapper");
let preprocessor = if let Some(pre_spread) = pre_spread {
format!(
@@ -664,6 +682,8 @@ except BaseException as e:
);
write_file(job_dir, "wrapper.py", &wrapper_content)?;
tracing::debug!("Finished writing wrapper");
let client = client.get_authed().await;
let mut reserved_variables = get_reserved_variables(job, &client.token, db).await?;
let additional_python_paths_folders = additional_python_paths.iter().join(":");
@@ -749,7 +769,8 @@ mount {{
.stdout(Stdio::piped())
.stderr(Stdio::piped());
#[cfg(windows)] {
#[cfg(windows)]
{
python_cmd.env("SystemRoot", SYSTEM_ROOT.as_str());
python_cmd.env("USERPROFILE", crate::USERPROFILE_ENV.as_str());
}
@@ -1093,7 +1114,7 @@ async fn handle_python_deps(
let mut venv_path = handle_python_reqs(
requirements
.split("\n")
.filter(|x| !x.starts_with("--"))
.filter(|x| !x.starts_with("--") && !x.trim().is_empty())
.collect(),
job_id,
w_id,
@@ -1239,29 +1260,28 @@ async fn spawn_uv_install(
#[cfg(unix)]
{
if no_uv_install {
let mut flock_cmd = Command::new(FLOCK_PATH.as_str());
flock_cmd
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(envs)
.args([
"-x",
&format!(
"{}/{}-{}.lock",
LOCK_CACHE_DIR,
if no_uv_install { "pip" } else { "py311" },
fssafe_req
),
"--command",
&command_args.join(" "),
])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(flock_cmd, FLOCK_PATH.as_str()).await
let mut flock_cmd = Command::new(FLOCK_PATH.as_str());
flock_cmd
.env_clear()
.envs(PROXY_ENVS.clone())
.envs(envs)
.args([
"-x",
&format!(
"{}/{}-{}.lock",
LOCK_CACHE_DIR,
if no_uv_install { "pip" } else { "py311" },
fssafe_req
),
"--command",
&command_args.join(" "),
])
.stdout(Stdio::piped())
.stderr(Stdio::piped());
start_child_process(flock_cmd, FLOCK_PATH.as_str()).await
} else {
let mut cmd = Command::new(command_args[0]);
cmd
.env_clear()
cmd.env_clear()
.envs(PROXY_ENVS.clone())
.envs(envs)
.args(&command_args[1..])
@@ -1321,7 +1341,6 @@ pub async fn handle_python_reqs(
mut no_uv_install: bool,
is_ansible: bool,
) -> error::Result<Vec<String>> {
let counter_arc = Arc::new(tokio::sync::Mutex::new(0));
// Append logs with line like this:
// [9/21] + requests==2.32.3 << (S3) | in 57ms
@@ -1405,7 +1424,6 @@ pub async fn handle_python_reqs(
.map(handle_ephemeral_token),
);
// Prepare NSJAIL
if !*DISABLE_NSJAIL {
let _ = write_file(
@@ -1438,7 +1456,7 @@ pub async fn handle_python_reqs(
let mut in_cache = vec![];
for req in requirements {
// Ignore python version annotation backed into lockfile
if req.starts_with('#') {
if req.starts_with('#') || req.starts_with('-') || req.trim().is_empty() {
continue;
}
// TODO: Remove
@@ -1461,12 +1479,19 @@ pub async fn handle_python_reqs(
}
}
if in_cache.len() > 0 {
append_logs(&job_id, w_id, format!("\nenv deps from local cache: {}\n", in_cache.join(", ")), db).await;
append_logs(
&job_id,
w_id,
format!("\nenv deps from local cache: {}\n", in_cache.join(", ")),
db,
)
.await;
}
let (kill_tx, ..) = tokio::sync::broadcast::channel::<()>(1);
let kill_rxs: Vec<tokio::sync::broadcast::Receiver<()>> =
(0..req_with_penv.len()).map(|_| kill_tx.subscribe()).collect();
let kill_rxs: Vec<tokio::sync::broadcast::Receiver<()>> = (0..req_with_penv.len())
.map(|_| kill_tx.subscribe())
.collect();
// ________ Read comments at the end of the function to get more context
let (_done_tx, mut done_rx) = tokio::sync::mpsc::channel::<()>(1);
@@ -1506,21 +1531,21 @@ pub async fn handle_python_reqs(
if canceled {
tracing::info!(
// If there is listener on other side,
// If there is listener on other side,
workspace_id = %w_id_2,
"cancelling installations",
);
if let Err(ref e) = kill_tx.send(()){
tracing::error!(
// If there is listener on other side,
// If there is listener on other side,
workspace_id = %w_id_2,
"failed to send done: Probably receiving end closed too early or have not opened yet\n{}",
// If there is no listener, it will be dropped safely
e
);
}
}
}
}
// Once done_tx is dropped, this will be fired
_ = done_rx.recv() => break
@@ -1555,9 +1580,15 @@ pub async fn handle_python_reqs(
// Do we use Nsjail?
if !*DISABLE_NSJAIL {
logs.push_str(&format!("\nStarting isolated installation... ({} tasks in parallel) \n", parallel_limit));
logs.push_str(&format!(
"\nStarting isolated installation... ({} tasks in parallel) \n",
parallel_limit
));
} else {
logs.push_str(&format!("\nStarting installation... ({} tasks in parallel) \n", parallel_limit));
logs.push_str(&format!(
"\nStarting installation... ({} tasks in parallel) \n",
parallel_limit
));
}
append_logs(&job_id, w_id, logs, db).await;
}
@@ -1617,13 +1648,12 @@ pub async fn handle_python_reqs(
tokio::select! {
// Cancel was called on the job
_ = kill_rx.recv() => return Err(anyhow::anyhow!("S3 pull was canceled")),
pull = pull_from_tar(os, venv_p.clone(), no_uv_install) => {
if let Err(e) = pull {
tracing::info!(
workspace_id = %w_id,
"No tarball was found on S3 or different problem occured {job_id}:\n{e}",
);
);
} else {
print_success(
true,
@@ -1653,12 +1683,12 @@ pub async fn handle_python_reqs(
no_uv_install,
).await {
Ok(r) => r,
Err(e) => {
Err(e) => {
append_logs(
&job_id,
w_id,
format!(
"\nError while spawning proccess:\n{e}",
"\nError while spawning proccess:\n{e}",
),
db,
)
@@ -1698,7 +1728,7 @@ pub async fn handle_python_reqs(
&job_id,
w_id,
format!(
"\nError while installing {}:\n{buf}",
"\nError while installing {}:\n{buf}",
&req
),
db,
@@ -1757,7 +1787,10 @@ pub async fn handle_python_reqs(
let mut failed = false;
for (handle, (_, venv_p)) in handles.into_iter().zip(req_with_penv.into_iter()) {
if let Err(e) = handle.await.unwrap_or(Err(anyhow!("Problem by joining handle"))) {
if let Err(e) = handle
.await
.unwrap_or(Err(anyhow!("Problem by joining handle")))
{
failed = true;
tracing::warn!(
workspace_id = %w_id,
@@ -1778,19 +1811,11 @@ pub async fn handle_python_reqs(
if has_work {
let total_time = total_time.elapsed().as_millis();
append_logs(
&job_id,
w_id,
format!(
"\nenv set in {}ms",
total_time
),
db,
).await;
append_logs(&job_id, w_id, format!("\nenv set in {}ms", total_time), db).await;
}
// Usually done_tx will drop after this return
// If there is listener on other side,
// If there is listener on other side,
// it will be triggered
// If there is no listener, it will be dropped safely
return if failed {
backend/windmill-worker/src/result_processor.rs
+92 -28
View File
@@ -1,3 +1,6 @@
#[cfg(feature = "otel")]
use opentelemetry::trace::FutureExt;
use serde::Serialize;
use sqlx::{types::Json, Pool, Postgres};
use std::{
@@ -7,6 +10,9 @@ use std::{
Arc,
},
};
use tracing::{field, Instrument};
#[cfg(not(feature = "otel"))]
use windmill_common::otel_ee::FutureExt;
use uuid::Uuid;
@@ -85,7 +91,49 @@ pub fn start_background_processor(
JobKind::Dependencies | JobKind::FlowDependencies
);
handle_receive_completed_job(
let success = jc.success;
let span = tracing::span!(
tracing::Level::INFO,
"job_postprocessing",
job_id = %jc.job.id, root_job = field::Empty, workspace_id = %jc.job.workspace_id, worker = %worker_name,tag = %jc.job.tag,
// hostname = %hostname,
language = field::Empty,
script_path = field::Empty,
flow_step_id = field::Empty,
parent_job = field::Empty,
otel.name = field::Empty
);
let rj = if let Some(root_job) = jc.job.root_job {
root_job
} else {
jc.job.id
};
windmill_common::otel_ee::set_span_parent(&span, &rj);
if let Some(lg) = jc.job.language.as_ref() {
span.record("language", lg.as_str());
}
if let Some(step_id) = jc.job.flow_step_id.as_ref() {
span.record(
"otel.name",
format!("job_postprocessing {}", step_id).as_str(),
);
span.record("flow_step_id", step_id.as_str());
} else {
span.record("otel.name", "job postprocessing");
}
if let Some(parent_job) = jc.job.parent_job.as_ref() {
span.record("parent_job", parent_job.to_string().as_str());
}
if let Some(script_path) = jc.job.script_path.as_ref() {
span.record("script_path", script_path.as_str());
}
if let Some(root_job) = jc.job.root_job.as_ref() {
span.record("root_job", root_job.to_string().as_str());
}
let root_job = handle_receive_completed_job(
jc,
&base_internal_url,
&db,
@@ -96,8 +144,13 @@ pub fn start_background_processor(
#[cfg(feature = "benchmark")]
&mut bench,
)
.instrument(span)
.await;
if let Some(root_job) = root_job {
windmill_common::otel_ee::add_root_flow_job_to_otlp(&root_job, success);
}
if is_init_script_and_failure {
tracing::error!("init script errored, exiting");
killpill_tx.send(()).unwrap_or_default();
@@ -199,7 +252,11 @@ async fn send_job_completed(
token,
duration,
};
job_completed_tx.send(jc).await.expect("send job completed")
job_completed_tx
.send(jc)
.with_context(windmill_common::otel_ee::otel_ctx())
.await
.expect("send job completed")
}
pub async fn process_result(
@@ -271,6 +328,7 @@ pub async fn process_result(
token,
duration,
)
.with_context(windmill_common::otel_ee::otel_ctx())
.await;
Ok(true)
}
@@ -315,6 +373,7 @@ pub async fn process_result(
token,
duration,
)
.with_context(windmill_common::otel_ee::otel_ctx())
.await;
Ok(false)
}
@@ -330,7 +389,7 @@ pub async fn handle_receive_completed_job(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) {
) -> Option<Arc<QueuedJob>> {
let token = jc.token.clone();
let workspace = jc.job.workspace_id.clone();
let client = AuthedClient {
@@ -342,7 +401,7 @@ pub async fn handle_receive_completed_job(
let job = jc.job.clone();
let mem_peak = jc.mem_peak.clone();
let canceled_by = jc.canceled_by.clone();
if let Err(err) = process_completed_job(
match process_completed_job(
jc,
&client,
db,
@@ -355,26 +414,29 @@ pub async fn handle_receive_completed_job(
)
.await
{
handle_job_error(
db,
&client,
job.as_ref(),
mem_peak,
canceled_by,
err,
false,
same_worker_tx.clone(),
&worker_dir,
worker_name,
job_completed_tx,
#[cfg(feature = "benchmark")]
bench,
)
.await;
Err(err) => {
handle_job_error(
db,
&client,
job.as_ref(),
mem_peak,
canceled_by,
err,
false,
same_worker_tx.clone(),
&worker_dir,
worker_name,
job_completed_tx,
#[cfg(feature = "benchmark")]
bench,
)
.await;
None
}
Ok(r) => r,
}
}
#[tracing::instrument(name = "completed_job", level = "info", skip_all, fields(job_id = %job.id))]
pub async fn process_completed_job(
JobCompleted { job, result, mem_peak, success, cached_res_path, canceled_by, duration, .. }: JobCompleted,
client: &AuthedClient,
@@ -384,11 +446,11 @@ pub async fn process_completed_job(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> windmill_common::error::Result<()> {
) -> error::Result<Option<Arc<QueuedJob>>> {
if success {
// println!("bef completed job{:?}", SystemTime::now());
if let Some(cached_path) = cached_res_path {
save_in_cache(db, client, &job, cached_path.to_string(), &result).await;
save_in_cache(db, client, &job, cached_path, result.clone()).await;
}
let is_flow_step = job.is_flow_step;
@@ -414,8 +476,8 @@ pub async fn process_completed_job(
if is_flow_step {
if let Some(parent_job) = parent_job {
tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)");
update_flow_status_after_job_completion(
// tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)");
let r = update_flow_status_after_job_completion(
db,
client,
parent_job,
@@ -434,9 +496,10 @@ pub async fn process_completed_job(
)
.warn_after_seconds(10)
.await?;
add_time!(bench, "updated flow status END");
return Ok(r);
}
}
add_time!(bench, "updated flow status END");
} else {
let result = add_completed_job_error(
db,
@@ -454,7 +517,7 @@ pub async fn process_completed_job(
if job.is_flow_step {
if let Some(parent_job) = job.parent_job {
tracing::error!(parent_flow = %parent_job, subflow = %job.id, "process completed job error, updating flow status");
update_flow_status_after_job_completion(
let r = update_flow_status_after_job_completion(
db,
client,
parent_job,
@@ -473,10 +536,11 @@ pub async fn process_completed_job(
)
.warn_after_seconds(10)
.await?;
return Ok(r);
}
}
}
Ok(())
return Ok(None);
}
#[tracing::instrument(name = "job_error", level = "info", skip_all, fields(job_id = %job.id))]
backend/windmill-worker/src/snowflake_executor.rs
+1
View File
@@ -419,6 +419,7 @@ pub async fn do_snowflake(
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
Box::pin(futures::stream::once(async { 0 })),
)
.await?;
*mem_peak = (r.get().len() / 1000) as i32;
backend/windmill-worker/src/windmill-client.js
+6
View File
@@ -558,6 +558,9 @@ var $QueuedJob = {
"identity",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
],
},
schedule_path: {
@@ -697,6 +700,9 @@ var $CompletedJob = {
"identity",
"deploymentcallback",
"singlescriptflow",
"flowscript",
"flownode",
"appscript",
],
},
schedule_path: {
backend/windmill-worker/src/worker.rs
+153 -110
View File
@@ -6,7 +6,11 @@
* LICENSE-AGPL for a copy of the license.
*/
// #[cfg(feature = "otel")]
// use opentelemetry::{global, KeyValue};
use windmill_common::{
apps::AppScriptId,
auth::{fetch_authed_from_permissioned_as, JWTAuthClaims, JobPerms, JWT_SECRET},
scripts::PREVIEW_IS_TAR_CODEBASE_HASH,
utils::WarnAfterExt,
@@ -24,7 +28,7 @@ use const_format::concatcp;
#[cfg(feature = "prometheus")]
use prometheus::IntCounter;
use tracing::Instrument;
use tracing::{field, Instrument};
#[cfg(feature = "prometheus")]
use windmill_common::METRICS_DEBUG_ENABLED;
#[cfg(feature = "prometheus")]
@@ -34,9 +38,8 @@ use reqwest::Response;
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use sqlx::{types::Json, Pool, Postgres};
use std::{
collections::{hash_map::DefaultHasher, HashMap},
collections::HashMap,
fs::DirBuilder,
hash::Hash,
sync::{
atomic::{AtomicBool, AtomicU16, Ordering},
Arc,
@@ -91,8 +94,8 @@ use crate::{
bash_executor::{handle_bash_job, handle_powershell_job},
bun_executor::handle_bun_job,
common::{
build_args_map, get_cached_resource_value_if_valid, get_reserved_variables, hash_args,
update_worker_ping_for_failed_init_script, OccupancyMetrics,
build_args_map, cached_result_path, get_cached_resource_value_if_valid,
get_reserved_variables, update_worker_ping_for_failed_init_script, OccupancyMetrics,
},
csharp_executor::handle_csharp_job,
deno_executor::handle_deno_job,
@@ -102,18 +105,20 @@ use crate::{
handle_job_error,
job_logger::NO_LOGS_AT_ALL,
js_eval::{eval_fetch_timeout, transpile_ts},
mysql_executor::do_mysql,
pg_executor::do_postgresql,
php_executor::handle_php_job,
python_executor::handle_python_job,
result_processor::{process_result, start_background_processor},
rust_executor::handle_rust_job,
worker_flow::{handle_flow, update_flow_status_in_progress, Step},
worker_flow::{handle_flow, update_flow_status_in_progress},
worker_lockfiles::{
handle_app_dependency_job, handle_dependency_job, handle_flow_dependency_job,
},
};
#[cfg(feature = "mysql")]
use crate::mysql_executor::do_mysql;
use backon::ConstantBuilder;
use backon::{BackoffBuilder, Retryable};
@@ -722,7 +727,10 @@ fn add_outstanding_wait_time(
}.in_current_span());
}
#[tracing::instrument(name = "worker", level = "info", skip_all, fields(worker = %worker_name, hostname = %hostname))]
// struct WorkerMtrics {
// job_
// }
pub async fn run_worker(
db: &Pool<Postgres>,
hostname: &str,
@@ -738,6 +746,7 @@ pub async fn run_worker(
#[cfg(not(feature = "enterprise"))]
if !*DISABLE_NSJAIL {
tracing::warn!(
worker = %worker_name, hostname = %hostname,
"NSJAIL to sandbox process in untrusted environments is an enterprise feature but allowed to be used for testing purposes"
);
}
@@ -745,10 +754,10 @@ pub async fn run_worker(
let start_time = Instant::now();
let worker_dir = format!("{TMP_DIR}/{worker_name}");
tracing::debug!(worker_dir = %worker_dir, "Creating worker dir");
tracing::debug!(worker = %worker_name, hostname = %hostname, worker_dir = %worker_dir, "Creating worker dir");
if let Some(ref netrc) = *NETRC {
tracing::info!("Writing netrc at {}/.netrc", HOME_ENV.as_str());
tracing::info!(worker = %worker_name, hostname = %hostname, "Writing netrc at {}/.netrc", HOME_ENV.as_str());
write_file(&HOME_ENV, ".netrc", netrc).expect("could not write netrc");
}
@@ -963,6 +972,14 @@ pub async fn run_worker(
None
};
// let worker_resource = &[
// KeyValue::new("hostname", hostname.to_string()),
// KeyValue::new("worker", worker_name.to_string()),
// ];
// // Create a meter from the above MeterProvider.
// let meter = global::meter("windmill");
// let counter = meter.u64_counter("jobs.execution").build();
let mut occupancy_metrics = OccupancyMetrics::new(start_time);
let mut jobs_executed = 0;
@@ -1018,6 +1035,7 @@ pub async fn run_worker(
IS_READY.store(true, Ordering::Relaxed);
tracing::info!(
worker = %worker_name, hostname = %hostname,
"listening for jobs, WORKER_GROUP: {}, config: {:?}",
*WORKER_GROUP,
WORKER_CONFIG.read().await
@@ -1053,7 +1071,7 @@ pub async fn run_worker(
if i_worker == 1 {
if let Err(e) = queue_init_bash_maybe(db, same_worker_tx.clone(), &worker_name).await {
killpill_tx.send(()).unwrap_or_default();
tracing::error!("Error queuing init bash script for worker {worker_name}: {e:#}");
tracing::error!(worker = %worker_name, hostname = %hostname, "Error queuing init bash script for worker {worker_name}: {e:#}");
return;
}
}
@@ -1082,11 +1100,12 @@ pub async fn run_worker(
let mut last_suspend_first = Instant::now();
let mut killed_but_draining_same_worker_jobs = false;
let mut killpill_rx2 = killpill_rx.resubscribe();
loop {
#[cfg(feature = "enterprise")]
{
if let Ok(_) = killpill_rx.try_recv() {
tracing::info!("killpill received on worker waiting for valid key");
tracing::info!(worker = %worker_name, hostname = %hostname, "killpill received on worker waiting for valid key");
job_completed_tx
.0
.send(SendResult::Kill)
@@ -1098,6 +1117,7 @@ pub async fn run_worker(
if !valid_key {
tracing::error!(
worker = %worker_name, hostname = %hostname,
"Invalid license key, workers require a valid license key, sleeping for 30s waiting for valid key to be set"
);
tokio::time::sleep(Duration::from_secs(10)).await;
@@ -1111,7 +1131,7 @@ pub async fn run_worker(
#[cfg(feature = "prometheus")]
if let Some(wk) = worker_busy.as_ref() {
wk.set(0);
tracing::debug!("set worker busy to 0");
tracing::debug!(worker = %worker_name, hostname = %hostname, "set worker busy to 0");
}
occupancy_metrics.running_job_started_at = None;
@@ -1123,7 +1143,7 @@ pub async fn run_worker(
.try_into()
.unwrap(),
);
tracing::debug!("set uptime metric");
tracing::debug!(worker = %worker_name, hostname = %hostname, "set uptime metric");
}
if last_ping.elapsed().as_secs() > NUM_SECS_PING {
@@ -1167,15 +1187,19 @@ pub async fn run_worker(
)
.notify(|err, dur| {
tracing::error!(
worker = %worker_name, hostname = %hostname,
"retrying updating worker ping in {dur:#?}, err: {err:#?}"
);
})
.sleep(tokio::time::sleep)
.await {
tracing::error!("failed to update worker ping, exiting: {}", e);
tracing::error!(
worker = %worker_name, hostname = %hostname,
"failed to update worker ping, exiting: {}", e);
killpill_tx.send(()).unwrap_or_default();
}
tracing::info!(
worker = %worker_name, hostname = %hostname,
"ping update, memory: container={}MB, windmill={}MB",
memory_usage.unwrap_or_default() / (1024 * 1024),
wm_memory_usage.unwrap_or_default() / (1024 * 1024)
@@ -1187,16 +1211,18 @@ pub async fn run_worker(
if (jobs_executed as u32 + vacuum_shift) % VACUUM_PERIOD == 0 {
let db2 = db.clone();
let current_span = tracing::Span::current();
let worker_name = worker_name.clone();
let hostname = hostname.to_string();
tokio::task::spawn(
(async move {
tracing::info!("vacuuming queue and completed_job");
tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue");
if let Err(e) = sqlx::query!("VACUUM (skip_locked) queue")
.execute(&db2)
.await
{
tracing::error!("failed to vacuum queue: {}", e);
tracing::error!(worker = %worker_name, hostname = %hostname, "failed to vacuum queue: {}", e);
}
tracing::info!("vacuumed queue and completed_job");
tracing::info!(worker = %worker_name, hostname = %hostname, "vacuumed queue");
})
.instrument(current_span),
);
@@ -1232,6 +1258,7 @@ pub async fn run_worker(
if let Ok(same_worker_job) = same_worker_rx.try_recv() {
same_worker_queue_size.fetch_sub(1, Ordering::SeqCst);
tracing::debug!(
worker = %worker_name, hostname = %hostname,
"received {} from same worker channel",
same_worker_job.job_id
);
@@ -1244,6 +1271,7 @@ pub async fn run_worker(
.map_err(|_| Error::InternalErr("Impossible to fetch same_worker job".to_string()));
if r.is_err() && !same_worker_job.recoverable {
tracing::error!(
worker = %worker_name, hostname = %hostname,
"failed to fetch same_worker job on a non recoverable job, exiting"
);
job_completed_tx
@@ -1257,7 +1285,7 @@ pub async fn run_worker(
}
} else if let Ok(_) = killpill_rx.try_recv() {
if !killed_but_draining_same_worker_jobs {
tracing::info!("received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker);
tracing::info!(worker = %worker_name, hostname = %hostname, "received killpill for worker {}, jobs are not pulled anymore except same_worker jobs", i_worker);
killed_but_draining_same_worker_jobs = true;
job_completed_tx
.0
@@ -1268,10 +1296,10 @@ pub async fn run_worker(
continue;
} else if killed_but_draining_same_worker_jobs {
if job_completed_processor_is_done.load(Ordering::SeqCst) {
tracing::info!("all running jobs have completed and all completed jobs have been fully processed, exiting");
tracing::info!(worker = %worker_name, hostname = %hostname, "all running jobs have completed and all completed jobs have been fully processed, exiting");
break;
} else {
tracing::info!("there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting");
tracing::info!(worker = %worker_name, hostname = %hostname, "there may be same_worker jobs to process later, waiting for job_completed_processor to finish progressing all remaining flows before exiting");
tokio::time::sleep(Duration::from_millis(200)).await;
continue;
}
@@ -1296,7 +1324,7 @@ pub async fn run_worker(
if !agent_mode && duration_pull_s > 0.5 {
let empty = job.as_ref().is_ok_and(|x| x.0.is_none());
tracing::warn!("pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.5s ({duration_pull_s}), this is a sign that the database is VERY undersized for this load. empty: {empty}, err: {err_pull}");
#[cfg(feature = "prometheus")]
if empty {
if let Some(wp) = worker_pull_over_500_counter_empty.as_ref() {
@@ -1307,7 +1335,7 @@ pub async fn run_worker(
}
} else if !agent_mode && duration_pull_s > 0.1 {
let empty = job.as_ref().is_ok_and(|x| x.0.is_none());
tracing::warn!("pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
tracing::warn!(worker = %worker_name, hostname = %hostname, "pull took more than 0.1s ({duration_pull_s}) this is a sign that the database is undersized for this load. empty: {empty}, err: {err_pull}");
#[cfg(feature = "prometheus")]
if empty {
if let Some(wp) = worker_pull_over_100_counter_empty.as_ref() {
@@ -1361,7 +1389,7 @@ pub async fn run_worker(
last_executed_job = None;
jobs_executed += 1;
tracing::debug!("started handling of job {}", job.id);
tracing::debug!(worker = %worker_name, hostname = %hostname, "started handling of job {}", job.id);
if matches!(job.job_kind, JobKind::Script | JobKind::Preview) {
if !dedicated_workers.is_empty() {
@@ -1426,6 +1454,11 @@ pub async fn run_worker(
)
.await;
// counter.add(
// 1,
// worker_resource
// );
#[cfg(feature = "prometheus")]
let _timer = register_metric(
&WORKER_EXECUTION_DURATION,
@@ -1513,6 +1546,39 @@ pub async fn run_worker(
let PulledJob { job, raw_code, raw_lock, raw_flow } = job;
let arc_job = Arc::new(job);
add_time!(bench, "handle_queued_job START");
let span = tracing::span!(tracing::Level::INFO, "job",
job_id = %arc_job.id, root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, hostname = %hostname, tag = %arc_job.tag,
language = field::Empty,
script_path = field::Empty, flow_step_id = field::Empty, parent_job = field::Empty,
otel.name = field::Empty);
let rj = if let Some(root_job) = arc_job.root_job {
root_job
} else {
arc_job.id
};
if let Some(lg) = arc_job.language.as_ref() {
span.record("language", lg.as_str());
}
if let Some(step_id) = arc_job.flow_step_id.as_ref() {
span.record("otel.name", format!("job {}", step_id).as_str());
span.record("flow_step_id", step_id.as_str());
} else {
span.record("otel.name", "job");
}
if let Some(parent_job) = arc_job.parent_job.as_ref() {
span.record("parent_job", parent_job.to_string().as_str());
}
if let Some(script_path) = arc_job.script_path.as_ref() {
span.record("script_path", script_path.as_str());
}
if let Some(root_job) = arc_job.root_job.as_ref() {
span.record("root_job", root_job.to_string().as_str());
}
windmill_common::otel_ee::set_span_parent(&span, &rj);
// span.context().span().add_event_with_timestamp("job created".to_string(), arc_job.created_at.into(), vec![]);
match handle_queued_job(
arc_job.clone(),
raw_code,
@@ -1528,9 +1594,11 @@ pub async fn run_worker(
base_internal_url,
job_completed_tx.clone(),
&mut occupancy_metrics,
&mut killpill_rx2,
#[cfg(feature = "benchmark")]
&mut bench,
)
.instrument(span)
.await
{
Err(err) => {
@@ -1609,7 +1677,7 @@ pub async fn run_worker(
if let Some(secs) = *EXIT_AFTER_NO_JOB_FOR_SECS {
if let Some(lj) = last_executed_job {
if lj.elapsed().as_secs() > secs {
tracing::info!("no job for {} seconds, exiting", secs);
tracing::info!(worker = %worker_name, hostname = %hostname, "no job for {} seconds, exiting", secs);
break;
}
} else {
@@ -1640,12 +1708,12 @@ pub async fn run_worker(
});
}
Err(err) => {
tracing::error!("Failed to pull jobs: {}", err);
tracing::error!(worker = %worker_name, hostname = %hostname, "Failed to pull jobs: {}", err);
}
};
}
tracing::info!("worker {} exiting", worker_name);
tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exiting", worker_name);
#[cfg(feature = "benchmark")]
{
@@ -1660,20 +1728,20 @@ pub async fn run_worker(
if has_dedicated_workers {
for handle in dedicated_handles {
if let Err(e) = handle.await {
tracing::error!("error in dedicated worker waiting for it to end: {:?}", e)
tracing::error!(worker = %worker_name, hostname = %hostname, "error in dedicated worker waiting for it to end: {:?}", e)
}
}
tracing::info!("all dedicated workers have exited");
tracing::info!(worker = %worker_name, hostname = %hostname, "all dedicated workers have exited");
}
drop(job_completed_tx);
tracing::info!("waiting for job_completed_processor to finish processing remaining jobs");
tracing::info!(worker = %worker_name, hostname = %hostname, "waiting for job_completed_processor to finish processing remaining jobs");
if let Err(e) = send_result.await {
tracing::error!("error in awaiting send_result process: {e:?}")
}
tracing::info!("worker {} exited", worker_name);
tracing::info!("number of jobs executed: {}", jobs_executed);
tracing::info!(worker = %worker_name, hostname = %hostname, "worker {} exited", worker_name);
tracing::info!(worker = %worker_name, hostname = %hostname, "number of jobs executed: {}", jobs_executed);
}
async fn queue_init_bash_maybe<'c>(
@@ -1800,7 +1868,6 @@ pub struct PreviousResult<'a> {
pub previous_result: Option<&'a RawValue>,
}
#[tracing::instrument(name = "job", level = "info", skip_all, fields(job_id = %job.id))]
async fn handle_queued_job(
job: Arc<QueuedJob>,
raw_code: Option<String>,
@@ -1816,8 +1883,11 @@ async fn handle_queued_job(
base_internal_url: &str,
job_completed_tx: JobCompletedSender,
occupancy_metrics: &mut OccupancyMetrics,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> windmill_common::error::Result<bool> {
// Extract the active span from the context
if job.canceled {
return Err(Error::JsonErr(canceled_job_to_result(&job)));
}
@@ -1857,8 +1927,8 @@ async fn handle_queued_job(
}
}
let step = if job.is_flow_step {
let r = update_flow_status_in_progress(
if job.is_flow_step {
let _ = update_flow_status_in_progress(
db,
&job.workspace_id,
job.parent_job
@@ -1867,24 +1937,19 @@ async fn handle_queued_job(
)
.warn_after_seconds(5)
.await?;
Some(r)
} else {
if let Some(parent_job) = job.parent_job {
if let Err(e) = sqlx::query_scalar!(
"UPDATE queue SET flow_status = jsonb_set(jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], COALESCE(flow_status->$1, '{}'::jsonb)), array[$1, 'started_at'], to_jsonb(now()::text)) WHERE id = $2 AND workspace_id = $3",
&job.id.to_string(),
parent_job,
&job.workspace_id
)
.execute(db)
.warn_after_seconds(5)
.await {
tracing::error!("Could not update parent job started_at flow_status: {}", e);
}
} else if let Some(parent_job) = job.parent_job {
if let Err(e) = sqlx::query_scalar!(
"UPDATE queue SET flow_status = jsonb_set(jsonb_set(COALESCE(flow_status, '{}'::jsonb), array[$1], COALESCE(flow_status->$1, '{}'::jsonb)), array[$1, 'started_at'], to_jsonb(now()::text)) WHERE id = $2 AND workspace_id = $3",
&job.id.to_string(),
parent_job,
&job.workspace_id
)
.execute(db)
.warn_after_seconds(5)
.await {
tracing::error!("Could not update parent job started_at flow_status: {}", e);
}
None
};
}
let started = Instant::now();
let (raw_code, raw_lock, raw_flow) = match (raw_code, raw_lock, raw_flow) {
@@ -1903,68 +1968,25 @@ async fn handle_queued_job(
};
let cached_res_path = if job.cache_ttl.is_some() {
let version_hash = if let Some(h) = job.script_hash {
if matches!(job.job_kind, JobKind::FlowScript) {
format!("flowscript_{}", h.to_string())
} else {
format!("script_{}", h.to_string())
}
} else if let Some(rc) = raw_code.as_ref() {
use std::hash::Hasher;
let mut s = DefaultHasher::new();
rc.hash(&mut s);
format!("inline_{}", hex::encode(s.finish().to_be_bytes()))
} else if let Some(sqlx::types::Json(rc)) = raw_flow.as_ref() {
use std::hash::Hasher;
let mut s = DefaultHasher::new();
rc.get().hash(&mut s);
format!("flow_{}", hex::encode(s.finish().to_be_bytes()))
} else {
"none".to_string()
};
let args_hash = hash_args(
db,
&client.get_authed().await,
&job.workspace_id,
&job.id,
&job.args,
)
.await;
if job.is_flow_step && !matches!(job.job_kind, JobKind::Flow) {
let flow_path = sqlx::query_scalar!(
"SELECT script_path FROM queue WHERE id = $1",
&job.parent_job.unwrap()
Some(
cached_result_path(
db,
&client.get_authed().await,
&job,
raw_code.as_ref(),
raw_lock.as_ref(),
raw_flow.as_ref(),
)
.fetch_one(db)
.warn_after_seconds(5)
.await
.map_err(|e| {
Error::InternalErr(format!(
"Fetching script path from queue for caching purposes: {e:#}"
))
})?
.ok_or_else(|| Error::InternalErr(format!("Expected script_path")))?;
let step = match step.unwrap() {
Step::Step(i) => i.to_string(),
Step::PreprocessorStep => "preprocessor".to_string(),
Step::FailureStep => "failure".to_string(),
};
Some(format!(
"{flow_path}/cache/{version_hash}/{step}/{args_hash}"
))
} else if let Some(script_path) = &job.script_path {
Some(format!("{script_path}/cache/{version_hash}/{args_hash}"))
} else {
None
}
.await,
)
} else {
None
};
if let Some(cached_res_path) = cached_res_path.clone() {
if let Some(cached_res_path) = cached_res_path.as_ref() {
let authed_client = client.get_authed().await;
let cached_resource_value_maybe = get_cached_resource_value_if_valid(
let cached_result_maybe = get_cached_resource_value_if_valid(
db,
&authed_client,
&job.id,
@@ -1973,7 +1995,7 @@ async fn handle_queued_job(
)
.warn_after_seconds(5)
.await;
if let Some(cached_resource_value) = cached_resource_value_maybe {
if let Some(result) = cached_result_maybe {
{
let logs =
"Job skipped because args & path found in cache and not expired".to_string();
@@ -1981,8 +2003,8 @@ async fn handle_queued_job(
}
job_completed_tx
.send(JobCompleted {
job: job,
result: Arc::new(cached_resource_value),
job,
result,
mem_peak: 0,
canceled_by: None,
success: true,
@@ -2123,6 +2145,7 @@ async fn handle_queued_job(
&mut column_order,
&mut new_args,
occupancy_metrics,
killpill_rx,
)
.await;
occupancy_metrics.total_duration_of_running_jobs +=
@@ -2266,6 +2289,7 @@ async fn handle_code_execution_job(
column_order: &mut Option<Vec<String>>,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
killpill_rx: &mut tokio::sync::broadcast::Receiver<()>,
) -> error::Result<Box<RawValue>> {
let ContentReqLangEnvs {
content: inner_content,
@@ -2314,6 +2338,20 @@ async fn handle_code_execution_job(
codebase: None,
}
}
JobKind::AppScript => {
let (lockfile, content) = cache::app::fetch_script(
db,
AppScriptId(job.script_hash.unwrap_or(ScriptHash(0)).0),
)
.await?;
ContentReqLangEnvs {
content,
lockfile,
language: job.language.to_owned(),
envs: None,
codebase: None,
}
}
JobKind::DeploymentCallback => {
get_script_content_by_path(job.script_path.clone(), &job.workspace_id, db).await?
}
@@ -2336,6 +2374,10 @@ async fn handle_code_execution_job(
)
.await;
} else if language == Some(ScriptLang::Mysql) {
#[cfg(not(feature = "mysql"))]
return Err(Error::InternalErr("MySQL requires the mysql feature to be enabled".to_string()));
#[cfg(feature = "mysql")]
return do_mysql(
job,
&client,
@@ -2595,6 +2637,7 @@ mount {{
worker_name,
envs,
occupancy_metrics,
killpill_rx,
)
.await
}
backend/windmill-worker/src/worker_flow.rs
+274 -226
View File
@@ -6,13 +6,12 @@
* LICENSE-AGPL for a copy of the license.
*/
use std::collections::hash_map::DefaultHasher;
use std::collections::HashMap;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::Arc;
use std::time::Duration;
use crate::common::{hash_args, save_in_cache};
use crate::common::{cached_result_path, save_in_cache};
use crate::js_eval::{eval_timeout, IdContext};
use crate::{
AuthedClient, PreviousResult, SameWorkerPayload, SameWorkerSender, SendResult, JOB_TOKEN,
@@ -77,45 +76,34 @@ pub async fn update_flow_status_after_job_completion(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> error::Result<()> {
) -> error::Result<Option<Arc<QueuedJob>>> {
// this is manual tailrecursion because async_recursion blows up the stack
// todo!();
potentially_crash_for_testing();
let mut rec = update_flow_status_after_job_completion_internal(
db,
client,
let mut rec = RecUpdateFlowStatusAfterJobCompletion {
flow,
job_id_for_status,
w_id,
job_id_for_status: job_id_for_status.clone(),
success,
result,
unrecoverable,
same_worker_tx.clone(),
worker_dir,
stop_early_override,
false,
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
bench,
)
.await?;
while let Some(nrec) = rec {
skip_error_handler: false,
};
let mut unrecoverable = unrecoverable;
loop {
potentially_crash_for_testing();
rec = match update_flow_status_after_job_completion_internal(
let nrec = match update_flow_status_after_job_completion_internal(
db,
client,
nrec.flow,
&nrec.job_id_for_status,
rec.flow,
&rec.job_id_for_status,
w_id,
nrec.success,
nrec.result,
false,
rec.success,
rec.result,
unrecoverable,
same_worker_tx.clone(),
worker_dir,
nrec.stop_early_override,
nrec.skip_error_handler,
rec.stop_early_override,
rec.skip_error_handler,
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
@@ -125,12 +113,12 @@ pub async fn update_flow_status_after_job_completion(
{
Ok(j) => j,
Err(e) => {
tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", nrec.flow,&nrec.job_id_for_status);
tracing::error!("Error while updating flow status of {} after completion of {}, updating flow status again with error: {e:#}", rec.flow, &rec.job_id_for_status);
update_flow_status_after_job_completion_internal(
db,
client,
nrec.flow,
&nrec.job_id_for_status,
rec.flow,
&rec.job_id_for_status,
w_id,
false,
Arc::new(to_raw_value(&Json(&WrappedError {
@@ -139,8 +127,8 @@ pub async fn update_flow_status_after_job_completion(
true,
same_worker_tx.clone(),
worker_dir,
nrec.stop_early_override,
nrec.skip_error_handler,
rec.stop_early_override,
rec.skip_error_handler,
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
@@ -148,9 +136,33 @@ pub async fn update_flow_status_after_job_completion(
)
.await?
}
};
unrecoverable = false;
match nrec {
UpdateFlowStatusAfterJobCompletion::Done(job) => {
add_time!(bench, "update flow status internal END");
return Ok(Some(job));
}
UpdateFlowStatusAfterJobCompletion::Rec(nrec) => {
rec = nrec;
}
UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch => {
add_time!(bench, "update flow status internal END");
return Ok(None);
}
UpdateFlowStatusAfterJobCompletion::NotDone => {
add_time!(bench, "update flow status internal END");
return Ok(None);
}
}
}
Ok(())
}
pub enum UpdateFlowStatusAfterJobCompletion {
Rec(RecUpdateFlowStatusAfterJobCompletion),
Done(Arc<QueuedJob>),
NotDone,
NonLastParallelBranch,
}
pub struct RecUpdateFlowStatusAfterJobCompletion {
flow: uuid::Uuid,
@@ -188,7 +200,7 @@ pub async fn update_flow_status_after_job_completion_internal(
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
) -> error::Result<UpdateFlowStatusAfterJobCompletion> {
add_time!(bench, "update flow status internal START");
let (
should_continue_flow,
@@ -272,93 +284,96 @@ pub async fn update_flow_status_after_job_completion_internal(
let is_failure_step =
old_status.step >= old_status.modules.len() as i32 && old_status.modules.len() > 0;
let (mut stop_early, mut skip_if_stop_early, continue_on_error) = if let Some(se) =
stop_early_override
{
//do not stop early if module is a flow step
let step = match module_step {
Step::PreprocessorStep => None,
Step::FailureStep => None,
Step::Step(i) => Some(i),
};
let (mut stop_early, mut skip_if_stop_early, continue_on_error) =
if let Some(se) = stop_early_override {
//do not stop early if module is a flow step
let step = match module_step {
Step::PreprocessorStep => None,
Step::FailureStep => None,
Step::Step(i) => Some(i),
};
let is_flow = if let Some(_) = step {
#[derive(Deserialize)]
struct GetType<'j> { r#type: &'j str }
let is_flow = if let Some(_) = step {
#[derive(Deserialize)]
struct GetType<'j> {
r#type: &'j str,
}
current_module
.as_ref()
.map(|module| serde_json::from_str::<GetType>(module.value.get()).map(|v| v.r#type == "flow"))
.unwrap_or(Ok(false))
.unwrap_or(false)
} else {
false
};
if is_flow {
(false, false, false)
} else {
(true, se, false)
}
} else if is_failure_step || matches!(module_step, Step::PreprocessorStep) {
(false, false, false)
} else if let Some(current_module) = current_module.as_ref() {
let stop_early = success
&& !is_branch_all
&& if let Some(ref expr) = current_module
.stop_after_if
.as_ref()
.map(|x| x.expr.clone())
{
let all_iters = match &module_status {
FlowStatusModule::InProgress { flow_jobs: Some(flow_jobs), .. }
if expr.contains("all_iters") =>
{
Some(Arc::new(
retrieve_flow_jobs_results(db, w_id, flow_jobs).await?,
))
}
_ => None,
};
let args = sqlx::query_as::<_, RowArgs>(
"SELECT
args
FROM queue
WHERE id = $2",
)
.bind(old_status.step)
.bind(flow)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!("retrieval of args from state: {e:#}"))
})?;
compute_bool_from_expr(
&expr,
Marc::new(args.args.unwrap_or_default().0),
result.clone(),
all_iters,
None,
Some(client),
None,
None,
)
.await?
current_module
.as_ref()
.map(|module| {
serde_json::from_str::<GetType>(module.value.get())
.map(|v| v.r#type == "flow")
})
.unwrap_or(Ok(false))
.unwrap_or(false)
} else {
false
};
(
stop_early,
current_module
.stop_after_if
.as_ref()
.map(|x| x.skip_if_stopped)
.unwrap_or(false),
current_module.continue_on_error.unwrap_or(false),
)
} else {
(false, false, false)
};
if is_flow {
(false, false, false)
} else {
(true, se, false)
}
} else if is_failure_step || matches!(module_step, Step::PreprocessorStep) {
(false, false, false)
} else if let Some(current_module) = current_module.as_ref() {
let stop_early = success
&& !is_branch_all
&& if let Some(ref expr) = current_module
.stop_after_if
.as_ref()
.map(|x| x.expr.clone())
{
let all_iters =
match &module_status {
FlowStatusModule::InProgress {
flow_jobs: Some(flow_jobs), ..
} if expr.contains("all_iters") => Some(Arc::new(
retrieve_flow_jobs_results(db, w_id, flow_jobs).await?,
)),
_ => None,
};
let args = sqlx::query_as::<_, RowArgs>(
"SELECT
args
FROM queue
WHERE id = $2",
)
.bind(old_status.step)
.bind(flow)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!("retrieval of args from state: {e:#}"))
})?;
compute_bool_from_expr(
&expr,
Marc::new(args.args.unwrap_or_default().0),
result.clone(),
all_iters,
None,
Some(client),
None,
None,
)
.await?
} else {
false
};
(
stop_early,
current_module
.stop_after_if
.as_ref()
.map(|x| x.skip_if_stopped)
.unwrap_or(false),
current_module.continue_on_error.unwrap_or(false),
)
} else {
(false, false, false)
};
let skip_branch_failure = match module_status {
FlowStatusModule::InProgress {
@@ -420,7 +435,7 @@ pub async fn update_flow_status_after_job_completion_internal(
None
};
let nindex = if let Some(position) = position {
let nindex = if let Some(position) = position {
sqlx::query_scalar!(
"UPDATE queue
SET flow_status = JSONB_SET(
@@ -467,7 +482,7 @@ pub async fn update_flow_status_after_job_completion_internal(
None
};
let nindex = if let Some(position) = position {
let nindex = if let Some(position) = position {
sqlx::query_scalar!(
"UPDATE queue
SET flow_status = JSONB_SET(
@@ -603,7 +618,7 @@ pub async fn update_flow_status_after_job_completion_internal(
);
}
add_time!(bench, "non final parallel flow finished");
return Ok(None);
return Ok(UpdateFlowStatusAfterJobCompletion::NonLastParallelBranch);
}
}
FlowStatusModule::InProgress {
@@ -677,16 +692,15 @@ pub async fn update_flow_status_after_job_completion_internal(
}
}
if success || (flow_jobs.is_some() && (skip_loop_failures || skip_branch_failure)) {
let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some()) {
let is_skipped = if current_module.as_ref().is_some_and(|m| m.skip_if.is_some())
{
sqlx::query_scalar!(
"SELECT job_kind = 'identity' FROM completed_job WHERE id = $1",
job_id_for_status
)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!("error during skip check: {e:#}"))
})?
.map_err(|e| Error::InternalErr(format!("error during skip check: {e:#}")))?
.unwrap_or(false)
} else {
false
@@ -905,13 +919,15 @@ pub async fn update_flow_status_after_job_completion_internal(
.context("remove flow status retry")?;
}
let flow_job = sqlx::query_as::<_, PulledJob>("SELECT * FROM queue WHERE id = $1 AND workspace_id = $2")
.bind(flow)
.bind(w_id)
.fetch_optional(&mut *tx)
.await
.map_err(Into::<Error>::into)?
.ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
let flow_job = sqlx::query_as::<_, PulledJob>(
"SELECT * FROM queue WHERE id = $1 AND workspace_id = $2",
)
.bind(flow)
.bind(w_id)
.fetch_optional(&mut *tx)
.await
.map_err(Into::<Error>::into)?
.ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
tx.commit().await?;
let job_root = flow_job
@@ -927,7 +943,8 @@ pub async fn update_flow_status_after_job_completion_internal(
sqlx::query_scalar!(
"SELECT raw_flow AS \"raw_flow!: Json<Box<sqlx::types::JsonRawValue>>\"
FROM job WHERE id = $1 AND workspace_id = $2 LIMIT 1",
&flow_job.id, w_id
&flow_job.id,
w_id
)
.fetch_one(db)
.await
@@ -950,9 +967,16 @@ pub async fn update_flow_status_after_job_completion_internal(
flow_value
.as_ref()
.and_then(|value| match module_step {
Step::PreprocessorStep => value.preprocessor_module.as_ref().and_then(|m| m.retry.as_ref()),
Step::Step(i) => value.modules.get(i).as_ref().and_then(|m| m.retry.as_ref()),
Step::FailureStep => value.failure_module.as_ref().and_then(|m| m.retry.as_ref()),
Step::PreprocessorStep => value
.preprocessor_module
.as_ref()
.and_then(|m| m.retry.as_ref()),
Step::Step(i) => {
value.modules.get(i).as_ref().and_then(|m| m.retry.as_ref())
}
Step::FailureStep => {
value.failure_module.as_ref().and_then(|m| m.retry.as_ref())
}
})
.unwrap_or(&Retry::default()),
&old_status.retry,
@@ -964,7 +988,10 @@ pub async fn update_flow_status_after_job_completion_internal(
false
if !is_failure_step
&& !skip_error_handler
&& flow_value.as_ref().map(|v| v.failure_module.is_some()).unwrap_or(false) =>
&& flow_value
.as_ref()
.map(|v| v.failure_module.is_some())
.unwrap_or(false) =>
{
true
}
@@ -1032,29 +1059,17 @@ pub async fn update_flow_status_after_job_completion_internal(
reason: flow_job.canceled_reason.clone(),
}),
canceled_job_to_result(&flow_job),
worker_name,
worker_name,
true,
None,
)
.await?;
} else {
if flow_job.cache_ttl.is_some() && success {
let cached_res_path = {
let args_hash =
hash_args(db, client, w_id, job_id_for_status, &flow_job.args).await;
let flow_path = flow_job.script_path();
let version_hash = if let Some(sqlx::types::Json(s)) = raw_flow.as_ref() {
use std::hash::{Hash, Hasher};
let mut h = DefaultHasher::new();
s.get().hash(&mut h);
format!("flow_{}", hex::encode(h.finish().to_be_bytes()))
} else {
"flow_unknown".to_string()
};
format!("{flow_path}/cache/{version_hash}/{args_hash}")
};
let cached_res_path =
cached_result_path(db, client, &flow_job, None, None, raw_flow.as_ref()).await;
save_in_cache(db, client, &flow_job, cached_res_path, &nresult).await;
save_in_cache(db, client, &flow_job, cached_res_path, nresult.clone()).await;
}
fn result_has_recover_true(nresult: Arc<Box<RawValue>>) -> bool {
let recover = serde_json::from_str::<RecoveryObject>(nresult.get());
@@ -1108,7 +1123,7 @@ pub async fn update_flow_status_after_job_completion_internal(
Some(nresult.clone()),
same_worker_tx.clone(),
worker_dir,
job_completed_tx,
job_completed_tx,
)
.warn_after_seconds(10)
.await
@@ -1122,17 +1137,8 @@ pub async fn update_flow_status_after_job_completion_internal(
db,
)
.await;
let _ = add_completed_job_error(
db,
&flow_job,
0,
None,
e,
worker_name,
true,
None,
)
.await;
let _ = add_completed_job_error(db, &flow_job, 0, None, e, worker_name, true, None)
.await;
true
}
Ok(_) => false,
@@ -1148,23 +1154,25 @@ pub async fn update_flow_status_after_job_completion_internal(
if let Some(parent_job) = flow_job.parent_job {
tracing::info!(subflow_id = %flow_job.id, parent_id = %parent_job, "subflow is finished, updating parent flow status");
return Ok(Some(RecUpdateFlowStatusAfterJobCompletion {
flow: parent_job,
job_id_for_status: flow,
success: success && !is_failure_step,
result: nresult.clone(),
stop_early_override: if stop_early {
Some(skip_if_stop_early)
} else {
None
return Ok(UpdateFlowStatusAfterJobCompletion::Rec(
RecUpdateFlowStatusAfterJobCompletion {
flow: parent_job,
job_id_for_status: flow,
success: success && !is_failure_step,
result: nresult.clone(),
stop_early_override: if stop_early {
Some(skip_if_stop_early)
} else {
None
},
skip_error_handler: skip_error_handler || is_failure_step,
},
skip_error_handler: skip_error_handler || is_failure_step,
}));
));
}
}
Ok(None)
Ok(UpdateFlowStatusAfterJobCompletion::Done(flow_job))
} else {
Ok(None)
Ok(UpdateFlowStatusAfterJobCompletion::NotDone)
}
}
@@ -1292,7 +1300,7 @@ async fn compute_skip_branchall_failure<'c>(
// )))
// }
fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u16, Duration)> {
fn next_retry(retry: &Retry, status: &RetryStatus) -> Option<(u32, Duration)> {
(status.fail_count <= MAX_RETRY_ATTEMPTS)
.then(|| &retry)
.and_then(|retry| retry.interval(status.fail_count, false))
@@ -1495,8 +1503,7 @@ pub async fn handle_flow(
worker_dir: &str,
job_completed_tx: Sender<SendResult>,
) -> anyhow::Result<()> {
let flow = flow_value
.with_context(|| "Unable to parse flow definition")?;
let flow = flow_value.with_context(|| "Unable to parse flow definition")?;
let status = flow_job
.parse_flow_status()
.with_context(|| "Unable to parse flow status")?;
@@ -1511,14 +1518,15 @@ pub async fn handle_flow(
let schedule_path = flow_job.schedule_path.as_ref().unwrap();
let schedule =
get_schedule_opt(&mut tx, &flow_job.workspace_id, schedule_path).warn_after_seconds(5).await?;
let schedule = get_schedule_opt(&mut tx, &flow_job.workspace_id, schedule_path)
.warn_after_seconds(5)
.await?;
tx.commit().await?;
if let Some(schedule) = schedule {
if let Err(err) = handle_maybe_scheduled_job(
db,
db,
&flow_job,
&schedule,
flow_job.script_path.as_ref().unwrap(),
@@ -2247,14 +2255,16 @@ async fn push_next_flow_job(
} else {
let value = module.get_value();
match &value {
Ok(_) if matches!(value, Ok(FlowModuleValue::Identity)) || is_skipped => serde_json::from_str(
&serde_json::to_string(&PreviousResult {
previous_result: Some(&arc_last_job_result),
})
.unwrap(),
)
.map(Marc::new)
.map_err(|e| error::Error::InternalErr(format!("identity: {e:#}"))),
Ok(_) if matches!(value, Ok(FlowModuleValue::Identity)) || is_skipped => {
serde_json::from_str(
&serde_json::to_string(&PreviousResult {
previous_result: Some(&arc_last_job_result),
})
.unwrap(),
)
.map(Marc::new)
.map_err(|e| error::Error::InternalErr(format!("identity: {e:#}")))
}
Ok(
FlowModuleValue::Script { input_transforms, .. }
| FlowModuleValue::RawScript { input_transforms, .. }
@@ -2276,7 +2286,7 @@ async fn push_next_flow_job(
)
.await
.map(Marc::new)
},
}
Ok(_) => Ok(arc_flow_job_args.clone()),
Err(e) => {
return Err(error::Error::InternalErr(format!(
@@ -2533,7 +2543,9 @@ async fn push_next_flow_job(
};
tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed perms for job {i} of {len}");
let tag = if flow_job.tag == "flow" || flow_job.tag == format!("flow-{}", flow_job.workspace_id) {
let tag = if flow_job.tag == "flow"
|| flow_job.tag == format!("flow-{}", flow_job.workspace_id)
{
payload_tag.tag.clone()
} else {
Some(flow_job.tag.clone())
@@ -2950,7 +2962,7 @@ fn payload_from_modules<'a>(
if let Some(id) = modules_node {
return Some(JobPayload::FlowNode { id, path: path() });
}
add_virtual_items_if_necessary(&mut modules);
let mut failure_module = failure_module.cloned();
@@ -3094,7 +3106,7 @@ async fn compute_next_flow_transform(
ContinuePayload::SingleJob(payload),
NextStatus::NextStep,
))
},
}
FlowModuleValue::WhileloopFlow { modules, modules_node, .. } => {
// if it's a simple single step flow, we will collapse it as an optimization and need to pass flow_input as an arg
let is_simple = is_simple_modules(&modules, flow.failure_module.as_ref());
@@ -3196,15 +3208,21 @@ async fn compute_next_flow_transform(
.into_iter()
.filter_map(|i| {
let Some(payload) = payload_from_modules(
modules.clone(), modules_node, flow.failure_module.as_ref(), flow.same_worker,
modules.clone(),
modules_node,
flow.failure_module.as_ref(),
flow.same_worker,
|| format!("{}-{i}", status.step),
|| format!("{}/forloop-{i}", flow_job.script_path()),
true
true,
) else {
return None;
};
Some(JobPayloadWithTag {
payload, tag: None, delete_after_use, timeout: None
payload,
tag: None,
delete_after_use,
timeout: None,
})
})
.collect::<Vec<_>>();
@@ -3268,28 +3286,37 @@ async fn compute_next_flow_transform(
)))?,
};
let (modules, modules_node) = match branch {
BranchChosen::Default => (default, default_node),
BranchChosen::Branch { branch } => branches
.into_iter()
.nth(branch)
.map(|Branch { modules, modules_node, .. }| (modules, modules_node))
.ok_or_else(|| Error::BadRequest(format!("Unrecognized branch for BranchOne {status_module:?}")))?
.ok_or_else(|| {
Error::BadRequest(format!(
"Unrecognized branch for BranchOne {status_module:?}"
))
})?,
};
let Some(payload) = payload_from_modules(
modules, modules_node, flow.failure_module.as_ref(), flow.same_worker,
modules,
modules_node,
flow.failure_module.as_ref(),
flow.same_worker,
|| status.step.to_string(),
|| format!("{}/branchone-{}", flow_job.script_path(), status.step),
true
true,
) else {
return Ok(NextFlowTransform::EmptyInnerFlows);
};
Ok(NextFlowTransform::Continue(
ContinuePayload::SingleJob(JobPayloadWithTag {
payload, tag: None, delete_after_use, timeout: None
payload,
tag: None,
delete_after_use,
timeout: None,
}),
NextStatus::BranchChosen(branch),
))
@@ -3308,15 +3335,21 @@ async fn compute_next_flow_transform(
.enumerate()
.filter_map(|(i, Branch { modules, modules_node, .. })| {
let Some(payload) = payload_from_modules(
modules, modules_node, flow.failure_module.as_ref(), flow.same_worker,
modules,
modules_node,
flow.failure_module.as_ref(),
flow.same_worker,
|| format!("{}-{i}", status.step),
|| format!("{}/branchall-{}", flow_job.script_path(), i),
false
false,
) else {
return None;
};
Some(JobPayloadWithTag {
payload, tag: None, delete_after_use, timeout: None
payload,
tag: None,
delete_after_use,
timeout: None,
})
})
.collect::<Vec<_>>();
@@ -3358,20 +3391,36 @@ async fn compute_next_flow_transform(
let Branch { modules, modules_node, .. } = branches
.into_iter()
.nth(branch_status.branch)
.ok_or_else(|| Error::BadRequest(format!("Unrecognized branch for BranchAll {status_module:?}")))?;
.ok_or_else(|| {
Error::BadRequest(format!(
"Unrecognized branch for BranchAll {status_module:?}"
))
})?;
let Some(payload) = payload_from_modules(
modules, modules_node, flow.failure_module.as_ref(), flow.same_worker,
modules,
modules_node,
flow.failure_module.as_ref(),
flow.same_worker,
|| format!("{}-{}", status.step, branch_status.branch),
|| format!("{}/branchall-{}", flow_job.script_path(), branch_status.branch),
false
|| {
format!(
"{}/branchall-{}",
flow_job.script_path(),
branch_status.branch
)
},
false,
) else {
return Ok(NextFlowTransform::EmptyInnerFlows);
};
Ok(NextFlowTransform::Continue(
ContinuePayload::SingleJob(JobPayloadWithTag {
payload, tag: None, delete_after_use, timeout: None
payload,
tag: None,
delete_after_use,
timeout: None,
}),
NextStatus::NextBranchStep(NextBranch {
status: branch_status,
@@ -3404,25 +3453,25 @@ async fn next_loop_iteration(
| FlowModuleValue::FlowScript { input_transforms, .. }
| FlowModuleValue::Flow { input_transforms, .. } => {
Some(std::mem::take(input_transforms))
},
}
_ => None,
};
return Ok(NextFlowTransform::Continue(
ContinuePayload::SingleJob(
payload_from_simple_module(value, db, flow_job, module, Some(inner_path())).await?
payload_from_simple_module(value, db, flow_job, module, Some(inner_path())).await?,
),
NextStatus::NextLoopIteration {
next: ns,
simple_input_transforms,
},
NextStatus::NextLoopIteration { next: ns, simple_input_transforms },
));
}
let Some(payload) = payload_from_modules(
modules, modules_node, flow.failure_module.as_ref(), flow.same_worker,
modules,
modules_node,
flow.failure_module.as_ref(),
flow.same_worker,
|| format!("{}-{}", status.step, ns.index),
inner_path,
true
true,
) else {
return Ok(NextFlowTransform::EmptyInnerFlows);
};
@@ -3438,7 +3487,10 @@ async fn next_loop_iteration(
))
}
pub(super) fn is_simple_modules(modules: &Vec<FlowModule>, failure_module: Option<&Box<FlowModule>>) -> bool {
pub(super) fn is_simple_modules(
modules: &Vec<FlowModule>,
failure_module: Option<&Box<FlowModule>>,
) -> bool {
let is_simple = modules.len() == 1
&& modules[0].is_simple()
&& modules[0].sleep.is_none()
@@ -3685,11 +3737,7 @@ fn raw_script_to_payload(
}
fn flow_to_payload(path: String, delete_after_use: bool) -> JobPayloadWithTag {
let payload = JobPayload::Flow {
path,
dedicated_worker: None,
apply_preprocessor: false,
};
let payload = JobPayload::Flow { path, dedicated_worker: None, apply_preprocessor: false };
JobPayloadWithTag { payload, tag: None, delete_after_use, timeout: None }
}
backend/windmill-worker/src/worker_lockfiles.rs
+181 -61
View File
@@ -4,6 +4,7 @@ use std::path::{Component, Path, PathBuf};
use async_recursion::async_recursion;
use serde_json::value::RawValue;
use serde_json::{json, Value};
use sha2::Digest;
use sqlx::types::Json;
use uuid::Uuid;
use windmill_common::error::Error;
@@ -12,8 +13,9 @@ use windmill_common::flows::{FlowModule, FlowModuleValue, FlowNodeId};
use windmill_common::get_latest_deployed_hash_for_path;
use windmill_common::jobs::JobPayload;
use windmill_common::scripts::ScriptHash;
use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file};
use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, PythonAnnotations};
use windmill_common::{
apps::AppScriptId,
error::{self, to_anyhow},
flows::{add_virtual_items_if_necessary, FlowValue},
jobs::QueuedJob,
@@ -27,8 +29,10 @@ use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel};
use crate::common::OccupancyMetrics;
use crate::csharp_executor::generate_nuget_lockfile;
use crate::python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile};
use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile};
use crate::python_executor::{
create_dependencies_dir, handle_python_reqs, uv_pip_compile, USE_PIP_COMPILE, USE_PIP_INSTALL,
};
use crate::rust_executor::generate_cargo_lockfile;
use crate::{
bun_executor::gen_bun_lockfile,
deno_executor::generate_deno_lock,
@@ -648,7 +652,7 @@ pub async fn handle_flow_dependency_job(
// Compute a lite version of the flow value (`RawScript` => `FlowScript`).
let mut value_lite = flow.clone();
tx = reduce(
tx = reduce_flow(
tx,
&mut value_lite.modules,
&job_path,
@@ -1026,49 +1030,68 @@ async fn insert_flow_node<'c>(
flow: Option<&Json<Box<RawValue>>>,
) -> Result<(sqlx::Transaction<'c, sqlx::Postgres>, FlowNodeId)> {
let hash = {
use std::hash::{DefaultHasher, Hash, Hasher};
let mut hasher = DefaultHasher::new();
code.hash(&mut hasher);
lock.hash(&mut hasher);
flow.inspect(|flow| flow.get().hash(&mut hasher));
hasher.finish() as i64
let mut hasher = sha2::Sha256::new();
hasher.update(code.unwrap_or(&Default::default()));
hasher.update(lock.unwrap_or(&Default::default()));
hasher.update(flow.unwrap_or(&Default::default()).get());
format!("{:x}", hasher.finalize())
};
// Insert the flow node if it doesn't exist.
let id = sqlx::query_scalar!(
r#"
WITH existing AS (
SELECT id FROM flow_node
WHERE hash = $1 AND path = $2 AND workspace_id = $3
AND (code IS NOT DISTINCT FROM $4)
AND (lock IS NOT DISTINCT FROM $5)
AND (flow IS NOT DISTINCT FROM $6)
LIMIT 1
),
inserted AS (
INSERT INTO flow_node (hash, path, workspace_id, code, lock, flow)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT DO NOTHING
RETURNING id
)
SELECT id FROM existing
UNION ALL
SELECT id FROM inserted
INSERT INTO flow_node (path, workspace_id, hash_v2, lock, code, flow)
VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (path, workspace_id, hash_v2) DO UPDATE SET path = EXCLUDED.path -- trivial update to return the id
RETURNING id
"#,
hash,
path,
workspace_id,
code,
hash,
lock,
code,
flow as Option<&Json<Box<RawValue>>>
)
.fetch_one(&mut *tx)
.await?
.ok_or(error::Error::InternalErr("Failed to cache".to_string()))?;
.await?;
Ok((tx, FlowNodeId(id)))
}
async fn insert_app_script(
db: &sqlx::Pool<sqlx::Postgres>,
app: i64,
code: String,
lock: Option<String>,
) -> Result<AppScriptId> {
let code_sha256 = format!("{:x}", sha2::Sha256::digest(&code));
let hash = {
let mut hasher = sha2::Sha256::new();
hasher.update(app.to_le_bytes());
hasher.update(&code_sha256);
hasher.update(lock.as_ref().unwrap_or(&Default::default()));
format!("{:x}", hasher.finalize())
};
// Insert the app script if it doesn't exist.
sqlx::query_scalar!(
r#"
INSERT INTO app_script (app, hash, lock, code, code_sha256)
VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (hash) DO UPDATE SET app = EXCLUDED.app -- trivial update to return the id
RETURNING id
"#,
app,
hash,
lock,
code,
code_sha256
)
.fetch_one(db)
.await
.map(AppScriptId)
.map_err(Into::into)
}
async fn insert_flow_modules<'c>(
mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
path: &str,
@@ -1078,7 +1101,7 @@ async fn insert_flow_modules<'c>(
modules: &mut Vec<FlowModule>,
modules_node: &mut Option<FlowNodeId>,
) -> Result<sqlx::Transaction<'c, sqlx::Postgres>> {
tx = Box::pin(reduce(
tx = Box::pin(reduce_flow(
tx,
modules,
path,
@@ -1110,7 +1133,7 @@ async fn insert_flow_modules<'c>(
Ok(tx)
}
async fn reduce<'c>(
async fn reduce_flow<'c>(
mut tx: sqlx::Transaction<'c, sqlx::Postgres>,
modules: &mut Vec<FlowModule>,
path: &str,
@@ -1123,7 +1146,7 @@ async fn reduce<'c>(
let mut val =
serde_json::from_str::<FlowModuleValue>(module.value.get()).map_err(|err| {
Error::InternalErr(format!(
"reduce: Failed to parse flow module value: {}",
"reduce_flow: Failed to parse flow module value: {}",
err
))
})?;
@@ -1219,6 +1242,48 @@ async fn reduce<'c>(
Ok(tx)
}
async fn reduce_app(db: &sqlx::Pool<sqlx::Postgres>, value: &mut Value, app: i64) -> Result<()> {
match value {
Value::Object(object) => {
if let Some(Value::Object(script)) = object.get_mut("inlineScript") {
if script
.get("language")
.and_then(|x| x.as_str())
.is_some_and(|x| x == "frontend")
{
return Ok(());
}
// replace `content` with an empty string:
let Some(Value::String(code)) = script.get_mut("content").map(std::mem::take)
else {
return Err(error::Error::InternalErr(
"Missing `content` in inlineScript".to_string(),
));
};
// remove `lock`:
let lock = script.remove("lock").and_then(|x| match x {
Value::String(s) => Some(s),
_ => None,
});
let id = insert_app_script(db, app, code, lock).await?;
// insert the `id` into the `script` object:
script.insert("id".to_string(), json!(id.0));
} else {
for (_, value) in object {
Box::pin(reduce_app(db, value, app)).await?;
}
}
}
Value::Array(array) => {
for value in array {
Box::pin(reduce_app(db, value, app)).await?;
}
}
_ => {}
}
Ok(())
}
fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool {
if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative {
let anns = windmill_common::worker::TypeScriptAnnotations::parse(&content);
@@ -1404,11 +1469,12 @@ pub async fn handle_app_dependency_job(
.clone()
.ok_or_else(|| Error::InternalErr("App Dependency requires script hash".to_owned()))?
.0;
let value = sqlx::query_scalar!("SELECT value FROM app_version WHERE id = $1", id)
let record = sqlx::query!("SELECT app_id, value FROM app_version WHERE id = $1", id)
.fetch_optional(db)
.await?;
.await?
.map(|record| (record.app_id, record.value));
if let Some(value) = value {
if let Some((app_id, value)) = record {
let value = lock_modules_app(
value,
job,
@@ -1425,6 +1491,21 @@ pub async fn handle_app_dependency_job(
)
.await?;
// Compute a lite version of the app value (w/ `inlineScript.{lock,code}`).
let mut value_lite = value.clone();
reduce_app(db, &mut value_lite, app_id).await?;
if let Value::Object(object) = &mut value_lite {
object.insert("version".to_string(), json!(id));
}
sqlx::query!(
"INSERT INTO app_version_lite (id, value) VALUES ($1, $2)
ON CONFLICT (id) DO UPDATE SET value = EXCLUDED.value",
id,
sqlx::types::Json(to_raw_value(&value_lite)) as sqlx::types::Json<Box<RawValue>>,
)
.execute(db)
.await?;
// Re-check cancelation to ensure we don't accidentially override an app.
if sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job.id)
.fetch_optional(db)
@@ -1496,6 +1577,8 @@ async fn python_dep(
w_id: &str,
worker_dir: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
no_uv_compile: bool,
no_uv_install: bool,
) -> std::result::Result<String, Error> {
create_dependencies_dir(job_dir).await;
let req: std::result::Result<String, Error> = uv_pip_compile(
@@ -1508,7 +1591,7 @@ async fn python_dep(
worker_name,
w_id,
occupancy_metrics,
false,
no_uv_compile,
false,
)
.await;
@@ -1525,7 +1608,7 @@ async fn python_dep(
job_dir,
worker_dir,
occupancy_metrics,
false,
no_uv_install,
false,
)
.await;
@@ -1576,28 +1659,23 @@ async fn capture_dependency_job(
.join("\n")
};
python_dep(
reqs,
job_id,
mem_peak,
canceled_by,
job_dir,
db,
worker_name,
w_id,
worker_dir,
&mut Some(occupancy_metrics),
)
.await
}
ScriptLang::Ansible => {
if raw_deps {
return Err(Error::ExecutionErr(
"Raw dependencies not supported for ansible".to_string(),
));
let PythonAnnotations { no_uv, no_uv_install, no_uv_compile, .. } =
PythonAnnotations::parse(job_raw_code);
if no_uv || no_uv_install || no_uv_compile || *USE_PIP_COMPILE || *USE_PIP_INSTALL {
if let Err(e) = sqlx::query!(
r#"
INSERT INTO metrics (id, value)
VALUES ('no_uv_usage_py', $1)
"#,
serde_json::to_value("").map_err(to_anyhow)?
)
.execute(db)
.await
{
tracing::error!("Error inserting no_uv_usage_py to db: {:?}", e);
}
}
let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?;
let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default();
python_dep(
reqs,
@@ -1610,6 +1688,48 @@ async fn capture_dependency_job(
w_id,
worker_dir,
&mut Some(occupancy_metrics),
no_uv_compile | no_uv,
no_uv_install | no_uv,
)
.await
}
ScriptLang::Ansible => {
if raw_deps {
return Err(Error::ExecutionErr(
"Raw dependencies not supported for ansible".to_string(),
));
}
let (_logs, reqs, _) = windmill_parser_yaml::parse_ansible_reqs(job_raw_code)?;
let reqs = reqs.map(|r| r.python_reqs.join("\n")).unwrap_or_default();
if *USE_PIP_COMPILE || *USE_PIP_INSTALL {
if let Err(e) = sqlx::query!(
r#"
INSERT INTO metrics (id, value)
VALUES ('no_uv_usage_ansible', $1)
"#,
serde_json::to_value("").map_err(to_anyhow)?
)
.execute(db)
.await
{
tracing::error!("Error inserting no_uv_usage_ansible to db: {:?}", e);
};
}
python_dep(
reqs,
job_id,
mem_peak,
canceled_by,
job_dir,
db,
worker_name,
w_id,
worker_dir,
&mut Some(occupancy_metrics),
false,
false,
)
.await
}
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.435.1";
export const VERSION = "v1.437.1";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/main.ts
+1 -1
View File
@@ -60,7 +60,7 @@ export {
// }
// });
export const VERSION = "1.435.1";
export const VERSION = "1.437.1";
const command = new Command()
.name("wmill")
docker/DockerfileSlim
+1 -1
View File
@@ -20,7 +20,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
docker/DockerfileSlimEe
+1 -1
View File
@@ -19,7 +19,7 @@ RUN /usr/local/bin/python3 -m pip install pip-tools
# Install UV
RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh && mv /root/.cargo/bin/uv /usr/local/bin/uv
COPY --from=oven/bun:1.1.34 /usr/local/bin/bun /usr/bin/bun
COPY --from=oven/bun:1.1.38 /usr/local/bin/bun /usr/bin/bun
# add the docker client to call docker from a worker if enabled
COPY --from=docker:dind /usr/local/bin/docker /usr/local/bin/
flake.lock
Generated
+6 -6
View File
@@ -20,11 +20,11 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1732014248,
"narHash": "sha256-y/MEyuJ5oBWrWAic/14LaIr/u5E0wRVzyYsouYY3W6w=",
"lastModified": 1733759999,
"narHash": "sha256-463SNPWmz46iLzJKRzO3Q2b0Aurff3U1n0nYItxq7jU=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "23e89b7da85c3640bbc2173fe04f4bd114342367",
"rev": "a73246e2eef4c6ed172979932bc80e1404ba2d56",
"type": "github"
},
"original": {
@@ -61,11 +61,11 @@
"nixpkgs": "nixpkgs_2"
},
"locked": {
"lastModified": 1732156292,
"narHash": "sha256-XuTCME5ZausokOJ28AsIoayBVD1soscdoiKweT4VY50=",
"lastModified": 1733884434,
"narHash": "sha256-8GXR9kC07dyOIshAyfZhG11xfvBRSZzYghnZ2weOKJU=",
"owner": "oxalica",
"repo": "rust-overlay",
"rev": "2d484c7a0db32f2700e253160bcd2aaa6cdca3ba",
"rev": "d0483df44ddf0fd1985f564abccbe568e020ddf2",
"type": "github"
},
"original": {

Some files were not shown because too many files have changed in this diff Show More