diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 3f035f45c0..62ac9abafe 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.826.0" + ".": "1.827.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ce5ffcde3..1b157fa61d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [1.827.0](https://github.com/windmill-labs/windmill/compare/v1.826.0...v1.827.0) (2026-10-07) + + +### Features + +* audit instance settings writes, stable audit OTel target ([#11593](https://github.com/windmill-labs/windmill/issues/11593)) ([0792332](https://github.com/windmill-labs/windmill/commit/0792332859a88b1359ea1be64250d38dd071a0a9)) +* default new raw apps to react 19 and ask for the data setup ([#11586](https://github.com/windmill-labs/windmill/issues/11586)) ([818ddd5](https://github.com/windmill-labs/windmill/commit/818ddd5d31bd81507e274dd18022c1cdbd9be842)) +* export queue delay as OTel and Prometheus metrics ([#11588](https://github.com/windmill-labs/windmill/issues/11588)) ([7565349](https://github.com/windmill-labs/windmill/commit/75653496e542550039cbe5c413d8fcb9d395cea8)) +* pause a worker group without stopping its workers ([#11596](https://github.com/windmill-labs/windmill/issues/11596)) ([95c4b30](https://github.com/windmill-labs/windmill/commit/95c4b3096cf10c1fe59de7a1a6876dfb5f02c39f)) +* run AI decision steps on OpenAI's Decisions API ([#11590](https://github.com/windmill-labs/windmill/issues/11590)) ([046df9e](https://github.com/windmill-labs/windmill/commit/046df9e44db2f2fae3059ccd7a3d6c6955ee8971)) + + +### Bug Fixes + +* avoid URLSearchParams.keys() so the frontend check passes ([#11599](https://github.com/windmill-labs/windmill/issues/11599)) ([fc6c84c](https://github.com/windmill-labs/windmill/commit/fc6c84c6974bba1850e90a088fad5460c8c93ba3)) +* let recover: true from the error handler turn nested failures green ([#11559](https://github.com/windmill-labs/windmill/issues/11559)) ([03e484e](https://github.com/windmill-labs/windmill/commit/03e484e59757c0d4d6f74511d9aa5a1e2f94af67)) +* run dependency resolution and build steps under nsjail ([#11609](https://github.com/windmill-labs/windmill/issues/11609)) ([a238d01](https://github.com/windmill-labs/windmill/commit/a238d01ef6a8c285aef6886c974904f24339f7a2)) +* scope the python wheel cache to workspaces with their own registry ([#11597](https://github.com/windmill-labs/windmill/issues/11597)) ([14381ce](https://github.com/windmill-labs/windmill/commit/14381cefee6fb2b0ece52fbf0001c2ce61bde1a1)) +* type snowflake columns in database manager insert and alter forms ([#11578](https://github.com/windmill-labs/windmill/issues/11578)) ([f2fb156](https://github.com/windmill-labs/windmill/commit/f2fb15603b2e0d11bd5d04dd675212a5e4a7b9d7)) + ## [1.826.0](https://github.com/windmill-labs/windmill/compare/v1.825.0...v1.826.0) (2026-10-07) diff --git a/backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json b/backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json similarity index 54% rename from backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json rename to backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json index bb7e4fa36b..e03d995f15 100644 --- a/backend/.sqlx/query-ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e.json +++ b/backend/.sqlx/query-58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0.json @@ -1,11 +1,11 @@ { "db_name": "PostgreSQL", - "query": "SELECT EXISTS (SELECT 1 FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running)", + "query": "SELECT canceled_by IS NOT NULL FROM v2_job_queue WHERE id = $1 AND workspace_id = $2 AND running", "describe": { "columns": [ { "ordinal": 0, - "name": "exists", + "name": "?column?", "type_info": "Bool" } ], @@ -19,5 +19,5 @@ null ] }, - "hash": "ee769c78ec0a782de18911cdcb655aad93cf23321cd0f42f7ffa5348bfbea26e" + "hash": "58ea7e2c6b07ffc7ab7387b2ebb542cba3a5532c64a9594d2e0ccdf21dee90d0" } diff --git a/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json b/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json new file mode 100644 index 0000000000..ba917c82db --- /dev/null +++ b/backend/.sqlx/query-91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "91fdd565bd210f8aa4c6d5f5e309573aefaeb5dbc6ac07f5e664bc8fe2cb2683" +} diff --git a/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json b/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json new file mode 100644 index 0000000000..e1657e156b --- /dev/null +++ b/backend/.sqlx/query-a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1 FOR UPDATE", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "coalesce", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a6d355b3121f199f3827c05aaa7c8f780ab4c9d3d373d3cd0bb489e0d173040c" +} diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 61e705a11c..aeb7670d17 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -14829,7 +14829,7 @@ dependencies = [ [[package]] name = "windmill" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-nats", @@ -14918,7 +14918,7 @@ dependencies = [ [[package]] name = "windmill-ai" -version = "1.826.0" +version = "1.827.0" dependencies = [ "async-stream", "async-trait", @@ -14952,7 +14952,7 @@ dependencies = [ [[package]] name = "windmill-alerting" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -14965,7 +14965,7 @@ dependencies = [ [[package]] name = "windmill-api" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "argon2", @@ -15105,7 +15105,7 @@ dependencies = [ [[package]] name = "windmill-api-agent-workers" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15128,7 +15128,7 @@ dependencies = [ [[package]] name = "windmill-api-assets" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15145,7 +15145,7 @@ dependencies = [ [[package]] name = "windmill-api-auth" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15171,7 +15171,7 @@ dependencies = [ [[package]] name = "windmill-api-client" -version = "1.826.0" +version = "1.827.0" dependencies = [ "reqwest 0.12.28", "serde", @@ -15181,7 +15181,7 @@ dependencies = [ [[package]] name = "windmill-api-configs" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15198,7 +15198,7 @@ dependencies = [ [[package]] name = "windmill-api-debug" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "base64 0.22.1", @@ -15220,7 +15220,7 @@ dependencies = [ [[package]] name = "windmill-api-embeddings" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15243,7 +15243,7 @@ dependencies = [ [[package]] name = "windmill-api-flow-conversations" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15259,7 +15259,7 @@ dependencies = [ [[package]] name = "windmill-api-flows" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15281,7 +15281,7 @@ dependencies = [ [[package]] name = "windmill-api-groups" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15303,7 +15303,7 @@ dependencies = [ [[package]] name = "windmill-api-inputs" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15317,7 +15317,7 @@ dependencies = [ [[package]] name = "windmill-api-integration-tests" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-nats", @@ -15353,7 +15353,7 @@ dependencies = [ [[package]] name = "windmill-api-jobs" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15379,7 +15379,7 @@ dependencies = [ [[package]] name = "windmill-api-npm-proxy" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15407,7 +15407,7 @@ dependencies = [ [[package]] name = "windmill-api-openapi" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15429,7 +15429,7 @@ dependencies = [ [[package]] name = "windmill-api-schedule" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15449,7 +15449,7 @@ dependencies = [ [[package]] name = "windmill-api-scripts" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15491,7 +15491,7 @@ dependencies = [ [[package]] name = "windmill-api-settings" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15520,7 +15520,7 @@ dependencies = [ [[package]] name = "windmill-api-sse" -version = "1.826.0" +version = "1.827.0" dependencies = [ "lazy_static", "serde", @@ -15532,7 +15532,7 @@ dependencies = [ [[package]] name = "windmill-api-users" -version = "1.826.0" +version = "1.827.0" dependencies = [ "argon2", "axum 0.8.9", @@ -15556,7 +15556,7 @@ dependencies = [ [[package]] name = "windmill-api-workers" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15570,7 +15570,7 @@ dependencies = [ [[package]] name = "windmill-api-workspaces" -version = "1.826.0" +version = "1.827.0" dependencies = [ "axum 0.8.9", "chrono", @@ -15609,7 +15609,7 @@ dependencies = [ [[package]] name = "windmill-audit" -version = "1.826.0" +version = "1.827.0" dependencies = [ "chrono", "lazy_static", @@ -15624,7 +15624,7 @@ dependencies = [ [[package]] name = "windmill-autoscaling" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "axum 0.8.9", @@ -15643,7 +15643,7 @@ dependencies = [ [[package]] name = "windmill-common" -version = "1.826.0" +version = "1.827.0" dependencies = [ "aes-gcm", "aho-corasick", @@ -15757,7 +15757,7 @@ dependencies = [ [[package]] name = "windmill-dep-map" -version = "1.826.0" +version = "1.827.0" dependencies = [ "chrono", "futures", @@ -15777,7 +15777,7 @@ dependencies = [ [[package]] name = "windmill-git-sync" -version = "1.826.0" +version = "1.827.0" dependencies = [ "regex", "serde", @@ -15794,7 +15794,7 @@ dependencies = [ [[package]] name = "windmill-indexer" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "astral-tokio-tar", @@ -15821,7 +15821,7 @@ dependencies = [ [[package]] name = "windmill-jseval" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "futures", @@ -15838,7 +15838,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.826.0" +version = "1.827.0" dependencies = [ "itertools 0.14.0", "lazy_static", @@ -15854,7 +15854,7 @@ dependencies = [ [[package]] name = "windmill-mcp" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -15875,7 +15875,7 @@ dependencies = [ [[package]] name = "windmill-native-triggers" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -15906,7 +15906,7 @@ dependencies = [ [[package]] name = "windmill-oauth" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "arc-swap", @@ -15931,7 +15931,7 @@ dependencies = [ [[package]] name = "windmill-object-store" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-stream", @@ -15967,7 +15967,7 @@ dependencies = [ [[package]] name = "windmill-operator" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "futures", @@ -15985,7 +15985,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.826.0" +version = "1.827.0" dependencies = [ "convert_case 0.6.0", "serde", @@ -15994,7 +15994,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -16006,7 +16006,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "serde_json", @@ -16018,7 +16018,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "gosyn", @@ -16030,7 +16030,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -16042,7 +16042,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "serde_json", @@ -16054,7 +16054,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "nu-parser", @@ -16065,7 +16065,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16076,7 +16076,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -16088,7 +16088,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16099,7 +16099,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-recursion", @@ -16121,7 +16121,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "serde_json", @@ -16133,7 +16133,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -16147,7 +16147,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "convert_case 0.6.0", @@ -16164,7 +16164,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -16177,7 +16177,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "serde", @@ -16189,7 +16189,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -16207,7 +16207,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -16223,7 +16223,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "rustpython-ast", @@ -16239,7 +16239,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -16253,7 +16253,7 @@ dependencies = [ [[package]] name = "windmill-queue" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-recursion", @@ -16292,7 +16292,7 @@ dependencies = [ [[package]] name = "windmill-runtime-nativets" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "const_format", @@ -16332,7 +16332,7 @@ dependencies = [ [[package]] name = "windmill-sql-datatype-parser-wasm" -version = "1.826.0" +version = "1.827.0" dependencies = [ "getrandom 0.3.4", "wasm-bindgen", @@ -16343,7 +16343,7 @@ dependencies = [ [[package]] name = "windmill-store" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-recursion", @@ -16378,7 +16378,7 @@ dependencies = [ [[package]] name = "windmill-test-utils" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16402,7 +16402,7 @@ dependencies = [ [[package]] name = "windmill-trigger" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16435,7 +16435,7 @@ dependencies = [ [[package]] name = "windmill-trigger-amqp" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16462,7 +16462,7 @@ dependencies = [ [[package]] name = "windmill-trigger-azure" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16495,7 +16495,7 @@ dependencies = [ [[package]] name = "windmill-trigger-email" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16515,7 +16515,7 @@ dependencies = [ [[package]] name = "windmill-trigger-gcp" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16549,7 +16549,7 @@ dependencies = [ [[package]] name = "windmill-trigger-http" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16586,7 +16586,7 @@ dependencies = [ [[package]] name = "windmill-trigger-kafka" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16609,7 +16609,7 @@ dependencies = [ [[package]] name = "windmill-trigger-mqtt" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16633,7 +16633,7 @@ dependencies = [ [[package]] name = "windmill-trigger-nats" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-nats", @@ -16657,7 +16657,7 @@ dependencies = [ [[package]] name = "windmill-trigger-postgres" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16692,7 +16692,7 @@ dependencies = [ [[package]] name = "windmill-trigger-sqs" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16720,7 +16720,7 @@ dependencies = [ [[package]] name = "windmill-trigger-websocket" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-trait", @@ -16745,7 +16745,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "bitflags 2.13.2", @@ -16764,7 +16764,7 @@ dependencies = [ [[package]] name = "windmill-worker" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-once-cell", @@ -16883,7 +16883,7 @@ dependencies = [ [[package]] name = "windmill-worker-volumes" -version = "1.826.0" +version = "1.827.0" dependencies = [ "bytes", "futures", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 7808c0d621..06fdc001f0 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "windmill" -version = "1.826.0" +version = "1.827.0" authors.workspace = true edition.workspace = true @@ -88,7 +88,7 @@ members = [ exclude = ["./windmill-duckdb-ffi-internal", "./parsers/windmill-parser-wasm"] [workspace.package] -version = "1.826.0" +version = "1.827.0" authors = ["Ruben Fiszel "] edition = "2021" diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 5f22bddd84..fffcc99883 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -acb254ac509aff244849c09e50a8d39aae185138 +1600d49ad5513af8b54ad65688d3d1c7edc977d9 \ No newline at end of file diff --git a/backend/parsers/windmill-parser-wasm/Cargo.lock b/backend/parsers/windmill-parser-wasm/Cargo.lock index d7e33313c6..99d82bb43b 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.lock +++ b/backend/parsers/windmill-parser-wasm/Cargo.lock @@ -6216,7 +6216,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windmill-common" -version = "1.826.0" +version = "1.827.0" dependencies = [ "aho-corasick", "anyhow", @@ -6302,7 +6302,7 @@ dependencies = [ [[package]] name = "windmill-macros" -version = "1.826.0" +version = "1.827.0" dependencies = [ "proc-macro2", "quote", @@ -6314,7 +6314,7 @@ dependencies = [ [[package]] name = "windmill-parser" -version = "1.826.0" +version = "1.827.0" dependencies = [ "convert_case", "serde", @@ -6323,7 +6323,7 @@ dependencies = [ [[package]] name = "windmill-parser-bash" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -6335,7 +6335,7 @@ dependencies = [ [[package]] name = "windmill-parser-csharp" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "serde_json", @@ -6347,7 +6347,7 @@ dependencies = [ [[package]] name = "windmill-parser-go" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "gosyn", @@ -6359,7 +6359,7 @@ dependencies = [ [[package]] name = "windmill-parser-graphql" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -6371,7 +6371,7 @@ dependencies = [ [[package]] name = "windmill-parser-java" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "serde_json", @@ -6383,7 +6383,7 @@ dependencies = [ [[package]] name = "windmill-parser-nu" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "nu-parser", @@ -6394,7 +6394,7 @@ dependencies = [ [[package]] name = "windmill-parser-php" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6405,7 +6405,7 @@ dependencies = [ [[package]] name = "windmill-parser-py" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "itertools 0.14.0", @@ -6417,7 +6417,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-asset" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6428,7 +6428,7 @@ dependencies = [ [[package]] name = "windmill-parser-py-imports" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "async-recursion", @@ -6450,7 +6450,7 @@ dependencies = [ [[package]] name = "windmill-parser-r" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "serde_json", @@ -6462,7 +6462,7 @@ dependencies = [ [[package]] name = "windmill-parser-ruby" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -6476,7 +6476,7 @@ dependencies = [ [[package]] name = "windmill-parser-rust" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "convert_case", @@ -6493,7 +6493,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -6506,7 +6506,7 @@ dependencies = [ [[package]] name = "windmill-parser-sql-asset" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "serde", @@ -6518,7 +6518,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -6536,7 +6536,7 @@ dependencies = [ [[package]] name = "windmill-parser-ts-asset" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "serde-wasm-bindgen", @@ -6552,7 +6552,7 @@ dependencies = [ [[package]] name = "windmill-parser-wac" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "rustpython-ast", @@ -6568,7 +6568,7 @@ dependencies = [ [[package]] name = "windmill-parser-wasm" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "getrandom 0.2.17", @@ -6600,7 +6600,7 @@ dependencies = [ [[package]] name = "windmill-parser-yaml" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "lazy_static", @@ -6614,7 +6614,7 @@ dependencies = [ [[package]] name = "windmill-types" -version = "1.826.0" +version = "1.827.0" dependencies = [ "anyhow", "bitflags", diff --git a/backend/parsers/windmill-parser-wasm/Cargo.toml b/backend/parsers/windmill-parser-wasm/Cargo.toml index ba69f6e380..9ac7f81c7f 100644 --- a/backend/parsers/windmill-parser-wasm/Cargo.toml +++ b/backend/parsers/windmill-parser-wasm/Cargo.toml @@ -12,7 +12,7 @@ resolver = "2" members = ["."] [workspace.package] -version = "1.826.0" +version = "1.827.0" edition = "2021" authors = ["Ruben Fiszel "] diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index 677ecd11a2..8d0af4915d 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -427,6 +427,8 @@ pub async fn initial_load( native_mode, // an agent worker never reads its group's config, only its token object_store_cache_config: None, + // nor whether it is paused: the server refuses it jobs instead + paused: false, })); } } @@ -5689,8 +5691,13 @@ pub async fn reload_worker_config(db: &DB, tx: KillpillSender, kill_if_change: b .as_ref() .is_some_and(|dws| !dws.is_empty()); + // Pausing or resuming must not restart the workers, dedicated ones included: a reload + // whose only difference is the flag skips every restart below. + let only_pause_changed = wc.paused != config.paused + && WorkerConfig { paused: wc.paused, ..config.clone() } == **wc; + if **wc != config || has_dedicated { - if kill_if_change { + if kill_if_change && !only_pause_changed { if has_dedicated || wc.dedicated_worker != config.dedicated_worker || wc.dedicated_workers != config.dedicated_workers diff --git a/backend/tests/job_token_scopes.rs b/backend/tests/job_token_scopes.rs index 6af694bff0..f805168bfa 100644 --- a/backend/tests/job_token_scopes.rs +++ b/backend/tests/job_token_scopes.rs @@ -410,5 +410,22 @@ async fn test_oidc_claim_carries_the_presented_scopes(db: Pool) -> any )?; assert_eq!(claims["job_token_scopes"], expected, "{job}"); } + + // A soft-canceled job is still `running` until its worker kills it. + sqlx::query("UPDATE v2_job_queue SET canceled_by = 'test-user' WHERE id = $1") + .bind(Uuid::parse_str(OIDC_JOB)?) + .execute(&db) + .await?; + let resp = client + .post(format!("{base}/oidc/token/test-audience")) + .bearer_auth(job_token(&db, OIDC_JOB).await?) + .send() + .await?; + assert_eq!( + resp.status(), + StatusCode::FORBIDDEN, + "{}", + resp.text().await? + ); Ok(()) } diff --git a/backend/tests/worker_group_pause.rs b/backend/tests/worker_group_pause.rs new file mode 100644 index 0000000000..b528bbfd77 --- /dev/null +++ b/backend/tests/worker_group_pause.rs @@ -0,0 +1,71 @@ +//! In its own test binary: it flips the process-wide `WORKER_CONFIG`, which would stall every +//! other worker test sharing the process. + +use futures::StreamExt; +use sqlx::{Pool, Postgres}; +use std::time::Duration; +use windmill_common::{ + jobs::{JobPayload, RawCode}, + scripts::ScriptLang, + worker::{Connection, WORKER_CONFIG}, +}; +use windmill_test_utils::*; + +fn set_paused(paused: bool) { + let mut wc = (**WORKER_CONFIG.load()).clone(); + wc.paused = paused; + WORKER_CONFIG.store(std::sync::Arc::new(wc)); +} + +#[sqlx::test(fixtures("base"))] +async fn paused_worker_leaves_jobs_queued_until_resumed(db: Pool) -> anyhow::Result<()> { + initialize_tracing().await; + let server = ApiServer::start(db.clone()).await?; + let port = server.addr.port(); + + set_paused(true); + let id = RunJob::from(JobPayload::Code(RawCode { + hash: None, + content: "echo done".to_string(), + path: None, + lock: None, + language: ScriptLang::Bash, + cache_ttl: None, + cache_ignore_s3_path: None, + dedicated_worker: None, + concurrency_settings: windmill_common::runnable_settings::ConcurrencySettings::default() + .into(), + debouncing_settings: windmill_common::runnable_settings::DebouncingSettings::default(), + modules: None, + tag: None, + })) + .push(&db) + .await; + + let mut completed = listen_for_completed_jobs(&db).await; + in_test_worker( + Connection::Sql(db.clone()), + async { + tokio::time::sleep(Duration::from_secs(3)).await; + let running = + sqlx::query_scalar::<_, bool>("SELECT running FROM v2_job_queue WHERE id = $1") + .bind(id) + .fetch_optional(&db) + .await + .unwrap(); + assert_eq!( + running, + Some(false), + "a paused worker must not pull the job" + ); + + set_paused(false); + while completed.next().await != Some(id) {} + }, + port, + ) + .await; + + assert!(completed_job(id, &db).await.success); + Ok(()) +} diff --git a/backend/windmill-api-configs/src/lib.rs b/backend/windmill-api-configs/src/lib.rs index 7598021f00..f1f4792e71 100644 --- a/backend/windmill-api-configs/src/lib.rs +++ b/backend/windmill-api-configs/src/lib.rs @@ -224,12 +224,13 @@ async fn update_config( #[cfg(not(feature = "enterprise"))] let config = if name.starts_with("worker__") { - // In CE, only allow setting worker_tags, cache_clear, init_bash, and native_mode + // In CE, only allow setting worker_tags, cache_clear, init_bash, native_mode and paused serde_json::json!({ "worker_tags": config.get("worker_tags"), "cache_clear": config.get("cache_clear"), "init_bash": config.get("init_bash"), - "native_mode": config.get("native_mode") + "native_mode": config.get("native_mode"), + "paused": config.get("paused") }) } else { config @@ -272,6 +273,18 @@ async fn update_config( } let mut tx = db.begin().await?; + let was_paused = if name.starts_with("worker__") { + sqlx::query_scalar!( + "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1 FOR UPDATE", + &name + ) + .fetch_optional(&mut *tx) + .await? + .flatten() + .unwrap_or(false) + } else { + false + }; sqlx::query!( "INSERT INTO config (name, config) VALUES ($1, $2) ON CONFLICT (name) DO UPDATE SET config = EXCLUDED.config", &name, @@ -291,6 +304,17 @@ async fn update_config( ) .await?; tx.commit().await?; + + if name.starts_with("worker__") { + let paused = config.get("paused").and_then(|v| v.as_bool()) == Some(true); + if paused != was_paused { + windmill_common::feature_usage::log_feature_usage( + "worker_group", + "pause_toggle", + if paused { "pause" } else { "resume" }, + ); + } + } Ok(format!("Updated config {name}")) } diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 1c0ac068f3..51d2fbcb81 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.3" info: - version: 1.826.0 + version: 1.827.0 title: Windmill API contact: diff --git a/backend/windmill-common/src/instance_config.rs b/backend/windmill-common/src/instance_config.rs index a4164cd2c5..700ba71df5 100644 --- a/backend/windmill-common/src/instance_config.rs +++ b/backend/windmill-common/src/instance_config.rs @@ -963,6 +963,9 @@ pub struct WorkerGroupConfig { pub autoscaling: Option, #[serde(skip_serializing_if = "Option::is_none")] pub native_mode: Option, + /// Workers of the group pull no new job while this is set; queued jobs wait. + #[serde(skip_serializing_if = "Option::is_none")] + pub paused: Option, /// Object store this group's dependency cache uses instead of the instance one. Same shape /// as the instance `object_store_cache_config`. #[serde(skip_serializing_if = "Option::is_none")] diff --git a/backend/windmill-common/src/log_context.rs b/backend/windmill-common/src/log_context.rs index cdb1e27aea..c5cd191da8 100644 --- a/backend/windmill-common/src/log_context.rs +++ b/backend/windmill-common/src/log_context.rs @@ -49,6 +49,7 @@ pub struct LogContext { // Worker / job span (windmill-worker/src/worker.rs) pub worker: Option, + pub worker_group: Option, pub hostname: Option, pub tag: Option, pub job_id: Option, diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index e68b81e4de..e185d1d445 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -477,6 +477,7 @@ lazy_static::lazy_static! { env_vars: Default::default(), native_mode: false, object_store_cache_config: Default::default(), + paused: false, }); pub static ref WORKER_PULL_QUERIES: arc_swap::ArcSwap> = arc_swap::ArcSwap::from_pointee(vec![]); @@ -844,11 +845,14 @@ pub fn make_suspended_pull_query(tags: &[String]) -> String { } // pub async fn make_suspended pub async fn store_suspended_pull_query(wc: &WorkerConfig) { - if wc.worker_tags.len() == 0 { - tracing::error!("Empty tags in worker tags, skipping"); - return; - } - let query = make_suspended_pull_query(&wc.worker_tags); + // An empty tag list must clear the query: keeping the previous one would leave a reloaded + // worker pulling suspended jobs for tags it no longer listens to. + let query = if wc.worker_tags.is_empty() { + tracing::error!("Empty tags in worker tags, pulling no suspended jobs"); + String::new() + } else { + make_suspended_pull_query(&wc.worker_tags) + }; WORKER_SUSPENDED_PULL_QUERY.store(std::sync::Arc::new(query)); } @@ -922,6 +926,42 @@ pub async fn store_pull_query(wc: &WorkerConfig) { WORKER_PULL_QUERIES_FAIRNESS.store(std::sync::Arc::new(fairness_queries)); } +const WORKER_GROUP_PAUSED_CACHE_TTL: Duration = Duration::from_secs(10); + +lazy_static::lazy_static! { + static ref WORKER_GROUP_PAUSED_CACHE: quick_cache::sync::Cache = + quick_cache::sync::Cache::new(1000); +} + +/// Whether the config of `worker_group` is paused, as the server sees it. A worker that reads +/// its own config knows from `WORKER_CONFIG`; this is for the server deciding on behalf of +/// workers that do not, and is cached since they ask on every pull. +pub async fn is_worker_group_paused(db: &DB, worker_group: &str) -> bool { + if let Some((paused, at)) = WORKER_GROUP_PAUSED_CACHE.get(worker_group) { + if at.elapsed() < WORKER_GROUP_PAUSED_CACHE_TTL { + return paused; + } + } + let paused = match sqlx::query_scalar!( + "SELECT COALESCE(config->'paused' = 'true'::jsonb, false) FROM config WHERE name = $1", + format!("worker__{worker_group}") + ) + .fetch_optional(db) + .await + { + Ok(paused) => paused.flatten().unwrap_or(false), + Err(e) => { + tracing::error!("Could not read whether worker group {worker_group} is paused: {e:#}"); + return false; + } + }; + WORKER_GROUP_PAUSED_CACHE.insert( + worker_group.to_string(), + (paused, std::time::Instant::now()), + ); + paused +} + lazy_static::lazy_static! { pub static ref WINDMILL_DIR: String = { let dir = std::env::var("WINDMILL_DIR") @@ -2470,6 +2510,12 @@ pub async fn load_worker_config( .collect(); priority_tags_sorted.push(PriorityTags { priority: 0, tags: Vec::from_iter(all_tags_set) }); // push the tags that were not listed as high priority with a priority = 0 priority_tags_sorted.sort_by_key(|elt| Reverse(elt.priority)); // sort by priority DESC + + // The buckets come out of hash maps: without a fixed order two loads of the same + // config compare unequal, and a reload reads that as a config change. + for bucket in priority_tags_sorted.iter_mut() { + bucket.tags.sort(); + } } else { // if no priority is used, push all tags with a priority to 0 priority_tags_sorted.push(PriorityTags { priority: 0, tags: worker_tags.clone() }); @@ -2539,6 +2585,7 @@ pub async fn load_worker_config( env_vars: resolved_env_vars, native_mode, object_store_cache_config: config.object_store_cache_config, + paused: config.paused.unwrap_or(false), }) } @@ -2629,6 +2676,7 @@ pub struct WorkerConfigOpt { pub env_vars_allowlist: Option>, pub native_mode: Option, pub object_store_cache_config: Option, + pub paused: Option, } impl Default for WorkerConfigOpt { @@ -2648,6 +2696,7 @@ impl Default for WorkerConfigOpt { env_vars_allowlist: Default::default(), native_mode: Default::default(), object_store_cache_config: Default::default(), + paused: Default::default(), } } } @@ -2670,12 +2719,15 @@ pub struct WorkerConfig { /// in the group config. Raw JSON: `windmill-common` cannot depend on the object store crate /// that parses it, and comparing the raw value is what tells a reload the store changed. pub object_store_cache_config: Option, + /// The group pulls no job from the queue: its workers finish what they hold, keep pinging + /// and leave queued jobs for when it is resumed. + pub paused: bool, } impl std::fmt::Debug for WorkerConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, object_store_cache_config: {} }}", - self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode, + write!(f, "WorkerConfig {{ worker_tags: {:?}, priority_tags_sorted: {:?}, dedicated_worker: {:?}, dedicated_workers: {:?}, init_bash: {:?}, periodic_script_bash: {:?}, periodic_script_interval_seconds: {:?}, cache_clear: {:?}, additional_python_paths: {:?}, pip_local_dependencies: {:?}, env_vars: {:?}, native_mode: {:?}, paused: {:?}, object_store_cache_config: {} }}", + self.worker_tags, self.priority_tags_sorted, self.dedicated_worker, self.dedicated_workers, self.init_bash, self.periodic_script_bash, self.periodic_script_interval_seconds, self.cache_clear, self.additional_python_paths, self.pip_local_dependencies, self.env_vars.iter().map(|(k, v)| format!("{}: {}{} ({} chars)", k, &v[..3.min(v.len())], "***", v.len())).collect::>().join(", "), self.native_mode, self.paused, // holds bucket credentials self.object_store_cache_config.as_ref().map(|_| "***").unwrap_or("None")) } diff --git a/backend/windmill-runtime-nativets/src/lib.rs b/backend/windmill-runtime-nativets/src/lib.rs index 1c2a76cda5..75aacbcc64 100644 --- a/backend/windmill-runtime-nativets/src/lib.rs +++ b/backend/windmill-runtime-nativets/src/lib.rs @@ -929,8 +929,8 @@ pub async fn eval_fetch_timeout( for line in logged.as_deref().unwrap_or(&log).lines() { tracing::info!( target: "windmill:job_log", - job_id = ?job_id, - workspace_id = ?w_id_for_tracing, + job_id = %job_id, + workspace_id = %w_id_for_tracing, "{line}" ); if *OTEL_JOB_LOGS { diff --git a/backend/windmill-worker/nsjail/build.csharp.config.proto b/backend/windmill-worker/nsjail/build.csharp.config.proto new file mode 100644 index 0000000000..a926d923f2 --- /dev/null +++ b/backend/windmill-worker/nsjail/build.csharp.config.proto @@ -0,0 +1,128 @@ +name: "csharp restore/build" + +mode: ONCE +hostname: "csharp" +log_level: ERROR +time_limit: {TIMEOUT} + +disable_rl: true + +cwd: "{JOB_DIR}" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: false +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "{DOTNET_ROOT}" + dst: "{DOTNET_ROOT}" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +# Container runtimes bind exactly these 3 files as separate submounts over +# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable. +# Load-bearing -- do not remove as redundant with the /etc bind above. +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hostname" + dst: "/etc/hostname" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +{TMP_MOUNT_BLOCK} + +# project files, nuget.config, and the published output dir are all here; cwd of dotnet +mount { + src: "{JOB_DIR}" + dst: "{JOB_DIR}" + is_bind: true + rw: true +} + +# DOTNET_CLI_HOME + NUGET_PACKAGES (nuget/). Writable, like the install/run jails' +# shared caches. +mount { + src: "{CSHARP_CACHE_DIR}" + dst: "{CSHARP_CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "{TRACING_PROXY_CA_CERT_PATH}" + dst: "{TRACING_PROXY_CA_CERT_PATH}" + is_bind: true + mandatory: false +} + +envar: "HOME=/tmp" + +#{DEV} diff --git a/backend/windmill-worker/nsjail/lock.php.config.proto b/backend/windmill-worker/nsjail/lock.php.config.proto new file mode 100644 index 0000000000..d06c1699a6 --- /dev/null +++ b/backend/windmill-worker/nsjail/lock.php.config.proto @@ -0,0 +1,124 @@ +name: "php composer install" + +mode: ONCE +hostname: "php" +log_level: ERROR +time_limit: {TIMEOUT} + +disable_rl: true + +cwd: "{JOB_DIR}" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: false +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +# Container runtimes bind exactly these 3 files as separate submounts over +# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable. +# Load-bearing -- do not remove as redundant with the /etc bind above. +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hostname" + dst: "/etc/hostname" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +mount { + src: "/dev/random" + dst: "/dev/random" + is_bind: true +} + +{TMP_MOUNT_BLOCK} + +# composer.json/composer.lock live here and vendor/ is written here; cwd of composer +mount { + src: "{JOB_DIR}" + dst: "{JOB_DIR}" + is_bind: true + rw: true +} + +# COMPOSER_HOME: package download cache. Writable, like the install/run jails' +# shared caches. +mount { + src: "{COMPOSER_CACHE_DIR}" + dst: "{COMPOSER_CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "{TRACING_PROXY_CA_CERT_PATH}" + dst: "{TRACING_PROXY_CA_CERT_PATH}" + is_bind: true + mandatory: false +} + +{CA_MOUNT} + +envar: "HOME=/tmp" + +#{DEV} diff --git a/backend/windmill-worker/nsjail/lock.py.config.proto b/backend/windmill-worker/nsjail/lock.py.config.proto new file mode 100644 index 0000000000..14119c4089 --- /dev/null +++ b/backend/windmill-worker/nsjail/lock.py.config.proto @@ -0,0 +1,133 @@ +name: "python lock compile" + +mode: ONCE +hostname: "python" +log_level: ERROR +time_limit: {TIMEOUT} + +# Dependency resolution may invoke the PEP 517 build backend of an sdist whose +# metadata is not static, which runs arbitrary package code (e.g. setup.py). +# That build, and uv's bytecode compile pool, size thread/fd pools to the host's +# CPU count, so these mirror download.py.config.proto rather than the run sandbox. +rlimit_as: 8192 +rlimit_cpu: 1000 +rlimit_fsize: 1024 +rlimit_nofile: 10000 + +envar: "HOME=/tmp" +envar: "LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH" + +cwd: "{JOB_DIR}" + +clone_newnet: false +clone_newuser: {CLONE_NEWUSER} + +skip_setsid: true +keep_caps: true +keep_env: true +mount_proc: true + +mount { + src: "/bin" + dst: "/bin" + is_bind: true +} + +mount { + src: "/lib" + dst: "/lib" + is_bind: true +} + +mount { + src: "/lib64" + dst: "/lib64" + is_bind: true + mandatory: false +} + +mount { + src: "/usr" + dst: "/usr" + is_bind: true +} + +mount { + src: "/etc" + dst: "/etc" + is_bind: true +} + +# Container runtimes bind exactly these 3 files as separate submounts over +# /etc; nsjail's ro remount of /etc is non-recursive so they stay writable. +# Load-bearing -- do not remove as redundant with the /etc bind above. +mount { + src: "/etc/resolv.conf" + dst: "/etc/resolv.conf" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hosts" + dst: "/etc/hosts" + is_bind: true + mandatory: false +} + +mount { + src: "/etc/hostname" + dst: "/etc/hostname" + is_bind: true + mandatory: false +} + +mount { + src: "/dev/null" + dst: "/dev/null" + is_bind: true + rw: true +} + +mount { + src: "/dev/urandom" + dst: "/dev/urandom" + is_bind: true +} + +{TMP_MOUNT_BLOCK} + +# requirements.in -> requirements.txt, cwd of uv +mount { + src: "{JOB_DIR}" + dst: "{JOB_DIR}" + is_bind: true + rw: true +} + +# shared uv cache: resolution metadata + where sdist build backends run +# (builds-v0/). Writable, as the install jail's target already is; a poisoned +# shared cache is the known residual, not a new one. +mount { + src: "{UV_CACHE_DIR}" + dst: "{UV_CACHE_DIR}" + is_bind: true + rw: true +} + +mount { + src: "{PY_INSTALL_DIR}" + dst: "{PY_INSTALL_DIR}" + is_bind: true +} + +{INDEX_CERT_MOUNT} + +mount { + src: "{TRACING_PROXY_CA_CERT_PATH}" + dst: "{TRACING_PROXY_CA_CERT_PATH}" + is_bind: true + mandatory: false +} + +#{DEV} diff --git a/backend/windmill-worker/src/csharp_executor.rs b/backend/windmill-worker/src/csharp_executor.rs index d393171f7f..7d77f8703f 100644 --- a/backend/windmill-worker/src/csharp_executor.rs +++ b/backend/windmill-worker/src/csharp_executor.rs @@ -33,7 +33,7 @@ use crate::{ get_proxy_envs_for_lang, handle_child::handle_child, is_sandboxing_enabled, read_ee_registry_with_workspace_override, CSHARP_CACHE_DIR, - DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, + DISABLE_NUSER, DOTNET_PATH, HOME_ENV, NSJAIL_PATH, NUGET_CONFIG, PATH_ENV, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, TZ_ENV, }; #[cfg(feature = "csharp")] @@ -47,6 +47,9 @@ use crate::SYSTEM_ROOT; #[cfg(feature = "csharp")] const NSJAIL_CONFIG_RUN_CSHARP_CONTENT: &str = include_str!("../nsjail/run.csharp.config.proto"); +#[cfg(feature = "csharp")] +const NSJAIL_CONFIG_BUILD_CSHARP_CONTENT: &str = + include_str!("../nsjail/build.csharp.config.proto"); #[cfg(feature = "csharp")] #[cfg(windows)] @@ -124,6 +127,9 @@ pub async fn generate_nuget_lockfile( gen_cs_proj(code, job_dir, reqs, lines_to_remove)?; + // `dotnet restore` resolves and downloads packages but does not import their + // MSBuild `build/*.targets`, so it runs no package code — only `dotnet publish` + // (jailed below) does — so it runs unsandboxed. let mut gen_lockfile_cmd = Command::new(DOTNET_PATH.as_str()); gen_lockfile_cmd .current_dir(job_dir) @@ -384,31 +390,82 @@ async fn build_cs_proj( } } - let mut build_cs_cmd = Command::new(DOTNET_PATH.as_str()); - build_cs_cmd - .current_dir(job_dir) - .env_clear() - .env("PATH", PATH_ENV.as_str()) - .env("BASE_INTERNAL_URL", base_internal_url) - .env("HOME", HOME_ENV.as_str()) - .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) - .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) - .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") - .env("DOTNET_NOLOGO", "true") - .env("MSBUILDDISABLENODEREUSE", "1") - .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) - .args(vec![ - "publish", - "--configuration", - "Release", - "-o", - &format!("{job_dir}/out"), - "--no-self-contained", - "-p:PublishSingleFile=true", - "-p:IncludeNativeLibrariesForSelfExtract=true", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); + let out_dir = format!("{job_dir}/out"); + let publish_args = vec![ + "publish", + "--configuration", + "Release", + "-o", + out_dir.as_str(), + "--no-self-contained", + "-p:PublishSingleFile=true", + "-p:IncludeNativeLibrariesForSelfExtract=true", + ]; + // `dotnet publish` imports and runs the MSBuild targets/tasks of restored NuGet + // packages (a package's build/*.targets execute at build, not at restore), i.e. + // arbitrary package code. Confine it under nsjail when sandboxing is on, the same + // way the run step is; HOME is set to /tmp by the config, keep_env passes the + // explicit dotnet/proxy vars, and the job dir + shared nuget cache are writable. + // Without sandboxing, publish runs directly with no isolation. + // mut is required by the #[cfg(windows)] env block below; unused on unix. + #[allow(unused_mut)] + let mut build_cs_cmd = if is_sandboxing_enabled() { + std::fs::create_dir_all(&*CSHARP_CACHE_DIR)?; + let nsjail_timeout = resolve_nsjail_timeout(conn, w_id, *job_id, None).await; + write_file( + job_dir, + "build.config.proto", + NSJAIL_CONFIG_BUILD_CSHARP_CONTENT + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{JOB_DIR}", job_dir) + .replace("{CSHARP_CACHE_DIR}", &*CSHARP_CACHE_DIR) + .replace("{DOTNET_ROOT}", DOTNET_ROOT.as_str()) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .as_str(), + )?; + let mut nsjail_args = vec!["--config", "build.config.proto", "--", DOTNET_PATH.as_str()]; + nsjail_args.extend(publish_args.iter().copied()); + let mut cmd = Command::new(NSJAIL_PATH.as_str()); + cmd.current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) + .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") + .env("DOTNET_NOLOGO", "true") + .env("MSBUILDDISABLENODEREUSE", "1") + .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) + .envs(PROXY_ENVS.clone()) + .envs(&*crate::worker::WHITELIST_ENVS) + .args(nsjail_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + cmd + } else { + let mut cmd = Command::new(DOTNET_PATH.as_str()); + cmd.current_dir(job_dir) + .env_clear() + .env("PATH", PATH_ENV.as_str()) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("HOME", HOME_ENV.as_str()) + .env("DOTNET_CLI_HOME", &*CSHARP_CACHE_DIR) + .env("NUGET_PACKAGES", format!("{}/nuget", *CSHARP_CACHE_DIR)) + .env("DOTNET_CLI_TELEMETRY_OPTOUT", "true") + .env("DOTNET_NOLOGO", "true") + .env("MSBUILDDISABLENODEREUSE", "1") + .env("DOTNET_ROOT", DOTNET_ROOT.as_str()) + .args(publish_args.clone()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + cmd + }; #[cfg(windows)] build_cs_cmd @@ -433,14 +490,20 @@ async fn build_cs_proj( .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), ); - let build_cs_process = start_child_process(build_cs_cmd, DOTNET_PATH.as_str(), true).await?; + let build_cs_executable = if is_sandboxing_enabled() { + NSJAIL_PATH.as_str() + } else { + DOTNET_PATH.as_str() + }; + let build_cs_process = start_child_process(build_cs_cmd, build_cs_executable, true).await?; handle_child( job_id, conn, mem_peak, canceled_by, build_cs_process, - false, + // jailed when sandboxing is on, so mem sampling finds the real process + is_sandboxing_enabled(), worker_name, w_id, "dotnet publish", diff --git a/backend/windmill-worker/src/handle_child.rs b/backend/windmill-worker/src/handle_child.rs index 22a2bf4e8b..fa06edf6bf 100644 --- a/backend/windmill-worker/src/handle_child.rs +++ b/backend/windmill-worker/src/handle_child.rs @@ -210,6 +210,53 @@ pub async fn handle_child( } } + impl KillReason { + /// Reports a kill once the process is gone: a span event on the enclosing span and, + /// through the logs bridge, a log record carrying the job's log context. + fn emit_canceled_event( + &self, + job_id: &Uuid, + timeout_reason: &str, + signal: &'static str, + detected_at: chrono::DateTime, + ) { + // `force` is only known for a soft cancel: a job completed under the worker + // may have been force canceled or completed by anything else server-side. + let (kill_reason, canceled_by, canceled_reason, force) = match self { + KillReason::TooManyLogs => return, + KillReason::Timeout { .. } => { + ("timeout", Some("timeout"), Some(timeout_reason), None) + } + KillReason::Cancelled(by) => ( + "cancelled", + by.as_ref().and_then(|x| x.username.as_deref()), + by.as_ref().and_then(|x| x.reason.as_deref()), + Some(false), + ), + KillReason::AlreadyCompleted => ("already_completed", None, None, None), + }; + let exited_at = chrono::Utc::now(); + let root_job = windmill_common::log_context::current_log_context() + .and_then(|c| c.root_job.clone()); + let rfc3339 = |t: chrono::DateTime| { + t.to_rfc3339_opts(chrono::SecondsFormat::Millis, true) + }; + tracing::info!( + job_id = %job_id, + root_job, + kill_reason, + canceled_by, + canceled_reason, + force, + kill_signal = signal, + cancel_detected_at = %rfc3339(detected_at), + process_exited_at = %rfc3339(exited_at), + kill_duration_ms = (exited_at - detected_at).num_milliseconds(), + "job.canceled" + ); + } + } + let (timeout_duration, timeout_warn_msg, timeout_source) = resolve_job_timeout(&conn, w_id, job_id, custom_timeout).await; if let Some(msg) = timeout_warn_msg { @@ -232,6 +279,7 @@ pub async fn handle_child( UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted, }, }; + let detected_at = chrono::Utc::now(); tx.send(()).expect("rx should never be dropped"); drop(tx); @@ -268,59 +316,78 @@ pub async fn handle_child( } }; - #[allow(unused_variables)] - if let Some(id) = child.id() { - if *MAX_WAIT_FOR_SIGINT > 0 { - #[cfg(any(target_os = "linux", target_os = "macos"))] - signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap(); + // Resolves once the process is reaped, to the last signal it was sent, or to `None` + // when the kill could not be confirmed. SIGINT and SIGTERM are only sent where the + // sends below are compiled in: elsewhere an exit during their wait is unsignaled. + let unix_signal = |s: &'static str| { + if cfg!(any(target_os = "linux", target_os = "macos")) { s } else { "none" } + }; + let kill = async { + #[allow(unused_variables)] + if let Some(id) = child.id() { + if *MAX_WAIT_FOR_SIGINT > 0 { + #[cfg(any(target_os = "linux", target_os = "macos"))] + signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap(); - for _ in 0..*MAX_WAIT_FOR_SIGINT { + for _ in 0..*MAX_WAIT_FOR_SIGINT { + if child.try_wait().is_ok_and(|x| x.is_some()) { + break; + } + sleep(Duration::from_secs(1)).await; + } if child.try_wait().is_ok_and(|x| x.is_some()) { - break; + set_reason.await; + return Ok(Some(unix_signal("SIGINT"))); } - sleep(Duration::from_secs(1)).await; } - if child.try_wait().is_ok_and(|x| x.is_some()) { - set_reason.await; - return Ok(Err(kill_reason)); + if sigterm { + #[cfg(any(target_os = "linux", target_os = "macos"))] + signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap(); + + for _ in 0..*MAX_WAIT_FOR_SIGTERM { + if child.try_wait().is_ok_and(|x| x.is_some()) { + break; + } + sleep(Duration::from_secs(1)).await; + } + if child.try_wait().is_ok_and(|x| x.is_some()) { + set_reason.await; + return Ok(Some(unix_signal("SIGTERM"))); + } } } - if sigterm { - #[cfg(any(target_os = "linux", target_os = "macos"))] - signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap(); - - for _ in 0..*MAX_WAIT_FOR_SIGTERM { - if child.try_wait().is_ok_and(|x| x.is_some()) { - break; + #[cfg(windows)] + { + let pid_to_kill = child.id(); + let killed = match kill_process_tree(pid_to_kill).await { + Ok(_) => { + tracing::debug!( + "successfully killed process tree with PID: {:?}", + pid_to_kill + ); + true } - sleep(Duration::from_secs(1)).await; - } - if child.try_wait().is_ok_and(|x| x.is_some()) { - set_reason.await; - return Ok(Err(kill_reason)); - } + Err(e) => { + tracing::error!("failed to kill process tree: {:?}", e); + false + } + }; + set_reason.await; + return Ok(killed.then_some("taskkill")); } - } - #[cfg(windows)] - { - let pid_to_kill = child.id(); - match kill_process_tree(pid_to_kill).await { - Ok(_) => tracing::debug!( - "successfully killed process tree with PID: {:?}", - pid_to_kill - ), - Err(e) => tracing::error!("failed to kill process tree: {:?}", e), - }; - set_reason.await; - return Ok(Err(kill_reason)); - } - #[cfg(unix)] - { - /* send SIGKILL and reap child process */ - let (_, kill) = future::join(set_reason, Box::into_pin(child.kill())).await; - kill.map(|()| Err(kill_reason)) + #[cfg(unix)] + { + /* send SIGKILL and reap child process */ + let (_, kill) = future::join(set_reason, Box::into_pin(child.kill())).await; + kill.map(|()| Some("SIGKILL")) + } + }; + let signal: io::Result> = kill.await; + if let Some(signal) = signal? { + kill_reason.emit_canceled_event(&job_id, &timeout_reason, signal, detected_at); } + Ok(Err(kill_reason)) }; let mut stream_result = Vec::new(); @@ -957,9 +1024,9 @@ where let conn = conn.clone(); // No tick at t=0: a job that finishes within the first interval issues none of the - // ping/metric statements below. Cancels are picked up by the first job ping (500 ms in, - // 2 s on agent workers). Memory is still sampled at start, without SQL: it is the only - // `mem_peak` reading a short job gets, and the completed job row reports it. + // ping/metric statements below. Cancels are picked up by the next job ping. Memory is still + // sampled at start, without SQL: it is the only `mem_peak` reading a short job gets, + // and the completed job row reports it. let mut interval = interval_at(Instant::now() + update_job_interval, update_job_interval); interval.set_missed_tick_behavior(MissedTickBehavior::Skip); *mem_peak = (*mem_peak).max(get_mem.next().await.unwrap_or(0)); @@ -1023,11 +1090,10 @@ where } } } - if matches!(conn, Connection::Http(_)) { - if i % 4 != 0 { - // only ping every 4th time (2s) on http agent mode - continue; - } + // An agent worker pays an HTTP round trip per ping, so it pings every 2 s for + // the first 10 s, then every 5 s. + if matches!(conn, Connection::Http(_)) && i % if i < 20 { 4 } else { 10 } != 0 { + continue; } let ping_job_status = ping_job_status(&conn, &job_id, Some(*mem_peak), if current_mem > 0 { Some(current_mem) } else { None }).await.unwrap_or_else(|e| { tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e); diff --git a/backend/windmill-worker/src/php_executor.rs b/backend/windmill-worker/src/php_executor.rs index 03328b82cb..8378455ef4 100644 --- a/backend/windmill-worker/src/php_executor.rs +++ b/backend/windmill-worker/src/php_executor.rs @@ -17,6 +17,7 @@ use windmill_queue::MiniPulledJob; use windmill_parser::{MainArgSignature, Typ}; use windmill_queue::{append_logs, CanceledBy}; +use crate::common::DEV_CONF_NSJAIL; use crate::{ common::{ build_command_with_isolation, check_executor_binary_exists, create_args_and_out_file, @@ -24,11 +25,13 @@ use crate::{ resolve_nsjail_tmp_mount_block, start_child_process, MaybeLock, OccupancyMetrics, }, handle_child::handle_child, - is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PHP_PATH, + is_sandboxing_enabled, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NUSER, NSJAIL_PATH, PATH_ENV, + PHP_PATH, PROXY_ENVS, TRACING_PROXY_CA_CERT_PATH, }; use windmill_common::client::AuthedClient; const NSJAIL_CONFIG_RUN_PHP_CONTENT: &str = include_str!("../nsjail/run.php.config.proto"); +const NSJAIL_CONFIG_LOCK_PHP_CONTENT: &str = include_str!("../nsjail/lock.php.config.proto"); lazy_static::lazy_static! { static ref RE: Regex = Regex::new(r"^//\s?(\S+)\s*$").unwrap(); @@ -199,15 +202,99 @@ pub async fn composer_install( write_file(job_dir, "composer.lock", lock)?; } - let mut child_cmd = Command::new(&*COMPOSER_PATH); - let args = vec!["install", "--no-dev", "--no-progress"]; - child_cmd - .current_dir(job_dir) - .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let child_process = start_child_process(child_cmd, &*COMPOSER_PATH, false).await?; + // `composer install` runs the plugin and script code of installed packages, so + // when sandboxing is on it is confined under nsjail like the run step. Inside the + // jail the environment is cleared down to what composer needs: PATH, COMPOSER_HOME + // (pointed at the mounted cache dir), the proxies, the operator-controlled + // WHITELIST_ENVS, and every other COMPOSER_* var from the worker env — Windmill has + // no managed composer-registry setting, so operators supply private Packagist auth + // and composer tuning (COMPOSER_AUTH, COMPOSER_PROCESS_TIMEOUT, …) through those. + // COMPOSER_CACHE_DIR is dropped so it can't point composer outside the jail, and a + // custom COMPOSER_CAFILE/COMPOSER_CAPATH is bind-mounted so TLS to private feeds + // keeps working. COMPOSER_PATH is Windmill-internal. Without sandboxing composer + // runs directly, with no isolation. + let composer_args = ["install", "--no-dev", "--no-progress"]; + let child_cmd = if is_sandboxing_enabled() { + // The cache dir is bind-mounted into the jail, so it must exist before + // nsjail builds the mount tree. + std::fs::create_dir_all(&*COMPOSER_CACHE_DIR)?; + let nsjail_timeout = resolve_nsjail_timeout(conn, w_id, *job_id, None).await; + // A custom CA bundle/dir lives outside the jail's mounts; bind it in so + // composer's TLS can read it (mandatory: false → skipped when the path is unset). + let ca_mount = ["COMPOSER_CAFILE", "COMPOSER_CAPATH"] + .iter() + .filter_map(|k| std::env::var(k).ok().filter(|v| !v.trim().is_empty())) + .map(|p| { + format!( + "mount {{\n src: \"{p}\"\n dst: \"{p}\"\n is_bind: true\n mandatory: false\n}}\n" + ) + }) + .collect::(); + write_file( + job_dir, + "lock.config.proto", + NSJAIL_CONFIG_LOCK_PHP_CONTENT + .replace("{TIMEOUT}", &nsjail_timeout) + .replace("{JOB_DIR}", job_dir) + .replace("{COMPOSER_CACHE_DIR}", &*COMPOSER_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!*DISABLE_NUSER).to_string()) + .replace("{TRACING_PROXY_CA_CERT_PATH}", &*TRACING_PROXY_CA_CERT_PATH) + .replace("{CA_MOUNT}", &ca_mount) + .replace("#{DEV}", DEV_CONF_NSJAIL) + .replace( + "{TMP_MOUNT_BLOCK}", + &resolve_nsjail_tmp_mount_block(job_dir).await, + ) + .as_str(), + )?; + + let composer_passthrough_envs: Vec<(String, String)> = std::env::vars() + .filter(|(k, _)| { + k.starts_with("COMPOSER_") + && k != "COMPOSER_HOME" + && k != "COMPOSER_CACHE_DIR" + && k != "COMPOSER_PATH" + }) + .collect(); + + let mut nsjail_args = vec![ + "--config", + "lock.config.proto", + "--", + COMPOSER_PATH.as_str(), + ]; + nsjail_args.extend(composer_args); + + let mut child_cmd = Command::new(NSJAIL_PATH.as_str()); + child_cmd + .current_dir(job_dir) + .env_clear() + // HOME is set to /tmp by the nsjail config; keep_env passes the rest. + .env("PATH", PATH_ENV.as_str()) + .envs(PROXY_ENVS.clone()) + .envs(&*crate::worker::WHITELIST_ENVS) + .envs(composer_passthrough_envs) + .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) + .args(nsjail_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + child_cmd + } else { + let mut child_cmd = Command::new(&*COMPOSER_PATH); + child_cmd + .current_dir(job_dir) + .env("COMPOSER_HOME", &*COMPOSER_CACHE_DIR) + .args(composer_args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + child_cmd + }; + let child_executable = if is_sandboxing_enabled() { + NSJAIL_PATH.as_str() + } else { + COMPOSER_PATH.as_str() + }; + let child_process = start_child_process(child_cmd, child_executable, false).await?; handle_child( job_id, @@ -215,7 +302,8 @@ pub async fn composer_install( mem_peak, canceled_by, child_process, - false, + // jailed when sandboxing is on, so mem sampling finds the real process + is_sandboxing_enabled(), worker_name, w_id, "composer install", diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index 3fce0fb938..7f2291099a 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -116,6 +116,7 @@ async fn handle_piptar_uploads(mut rx: tokio::sync::mpsc::UnboundedReceiver, + index_url: Option, + /// Caches filled through these URLs must not be shared with other workspaces. + ws_scoped: bool, +} + +impl PipIndexes { + async fn snapshot(w_id: &str) -> Self { + let overrides = crate::workspace_registry_overrides(w_id).await; + let ws_value = |key: &str| match overrides.get(key) { + Some(serde_json::Value::String(s)) => Some(s.clone()), + _ => None, + }; + // An empty value means unset, and an empty workspace override unsets the global one. + Self { + extra_index_url: ws_value("pip_extra_index_url") + .or(PIP_EXTRA_INDEX_URL.read().await.clone()) + .filter(|s| !s.trim().is_empty()), + index_url: ws_value("pip_index_url") + .or(PIP_INDEX_URL.read().await.clone()) + .filter(|s| !s.trim().is_empty()), + ws_scoped: !overrides.is_empty(), + } + } + + /// The `(extra index, index)` URLs to hand uv. Kept apart from [`Self::snapshot`]: it runs + /// `EPHEMERAL_TOKEN_CMD`, which a caller served from a cache has no reason to pay for. + async fn urls( + self, + job_id: &Uuid, + w_id: &str, + conn: &Connection, + ) -> (Option, Option) { + ( + read_ee_registry( + self.extra_index_url, + "pip extra index url", + job_id, + w_id, + conn, + ) + .await + .map(handle_ephemeral_token), + read_ee_registry(self.index_url, "pip index url", job_id, w_id, conn) + .await + .map(handle_ephemeral_token), + ) + } +} + lazy_static::lazy_static! { static ref PIP_SECRET_VARIABLE: Regex = Regex::new(r"\$\{PIP_SECRET:([^\s\}]+)\}").unwrap(); @@ -2527,28 +2634,18 @@ pub async fn handle_python_reqs( job_id ); - let pip_indexes = ( - read_ee_registry_with_workspace_override( - PIP_EXTRA_INDEX_URL.read().await.clone(), - "pip_extra_index_url", - "pip extra index url", - job_id, - w_id, - conn, - ) - .await - .map(handle_ephemeral_token), - read_ee_registry_with_workspace_override( - PIP_INDEX_URL.read().await.clone(), - "pip_index_url", - "pip index url", - job_id, - w_id, - conn, - ) - .await - .map(handle_ephemeral_token), - ); + let pip_indexes = PipIndexes::snapshot(w_id).await; + let ws_scoped = pip_indexes.ws_scoped; + let pip_indexes = pip_indexes.urls(job_id, w_id, conn).await; + // The wheel dir is shared by every workspace on the worker, and by every worker through + // the object store, under a name that says nothing of the index it was filled from. A + // workspace with its own registries gets its own, as its resolution cache does. The id + // is hashed: it is not restricted to characters safe in a file name. + let ws_dir_suffix = if ws_scoped { + format!("-ws-{}", &calculate_hash(w_id)[..16]) + } else { + String::new() + }; // Cached paths let mut req_with_penv: Vec<(String, String)> = vec![]; @@ -2577,7 +2674,7 @@ pub async fn handle_python_reqs( let py_prefix = &py_version.to_cache_dir(false); let venv_p = format!( - "{py_prefix}/{}", + "{py_prefix}/{}{ws_dir_suffix}", req.replace(' ', "").replace('/', "").replace(':', "") ); if metadata(venv_p.clone() + "/.valid.windmill").await.is_ok() { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 80ca37acb8..2acafa539f 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -949,6 +949,22 @@ pub async fn workspace_registry_cache_suffix(w_id: &str) -> String { } } +/// The registry overrides of a workspace, empty when it has none. +/// +/// One snapshot: registry settings reload under running jobs, so a caller whose cache key +/// has to agree with the registry values it used reads both from here rather than calling +/// [`workspace_registry_cache_suffix`] and a `read_ee_registry_*` helper separately. +pub async fn workspace_registry_overrides( + w_id: &str, +) -> std::collections::HashMap { + let registries = WORKSPACE_REGISTRIES.read().await; + registries + .as_ref() + .and_then(|m| m.get(w_id)) + .cloned() + .unwrap_or_default() +} + /// The name a build artifact is cached under, derived from `base` — the runnable's own /// cache-key input — and its inline modules. /// @@ -2005,6 +2021,7 @@ pub fn create_span_with_name( root_job = field::Empty, workspace_id = %arc_job.workspace_id, worker = %worker_name, + worker_group = %*WORKER_GROUP, hostname = field::Empty, tag = %arc_job.tag, language = field::Empty, @@ -2156,6 +2173,7 @@ pub fn log_context_for_job( job_id: Some(arc_job.id.to_string()), workspace_id: Some(arc_job.workspace_id.clone()), worker: Some(worker_name.to_string()), + worker_group: Some(WORKER_GROUP.clone()), tag: Some(arc_job.tag.clone()), job_kind: Some(arc_job.kind.as_str().to_string()), created_by: Some(arc_job.created_by.clone()), @@ -3406,6 +3424,10 @@ pub async fn run_worker( tokio::time::sleep(Duration::from_millis(200)).await; continue; } + } else if WORKER_CONFIG.load().paused { + // Checked after the same-worker channel: a flow this worker already runs + // still needs it for its remaining same-worker steps. + Ok(None) } else { match &conn { Connection::Sql(db) => { diff --git a/benchmarks/lib.ts b/benchmarks/lib.ts index 2b5215da99..281ba460b6 100644 --- a/benchmarks/lib.ts +++ b/benchmarks/lib.ts @@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts"; import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts"; import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts"; -export const VERSION = "v1.826.0"; +export const VERSION = "v1.827.0"; export async function login(email: string, password: string): Promise { return await windmill.UserService.login({ diff --git a/chat-sdk/package-lock.json b/chat-sdk/package-lock.json index fcc48449cf..692e5d33f5 100644 --- a/chat-sdk/package-lock.json +++ b/chat-sdk/package-lock.json @@ -1,12 +1,12 @@ { "name": "windmill-chat", - "version": "1.826.0", + "version": "1.827.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "windmill-chat", - "version": "1.826.0", + "version": "1.827.0", "license": "Apache-2.0", "devDependencies": { "@ai-sdk/react": "^4.0.102", diff --git a/chat-sdk/package.json b/chat-sdk/package.json index 33b6c43fa3..287636a9f8 100644 --- a/chat-sdk/package.json +++ b/chat-sdk/package.json @@ -1,7 +1,7 @@ { "name": "windmill-chat", "description": "Build chat interfaces on Windmill flows deployed in chat mode, from any frontend or raw app", - "version": "1.826.0", + "version": "1.827.0", "author": "Ruben Fiszel", "license": "Apache-2.0", "homepage": "https://github.com/windmill-labs/windmill/tree/main/chat-sdk#readme", diff --git a/cli/src/core/constants.ts b/cli/src/core/constants.ts index 6b59042168..d05c4e60a7 100644 --- a/cli/src/core/constants.ts +++ b/cli/src/core/constants.ts @@ -10,4 +10,4 @@ export const WM_FORK_PREFIX = "wm-fork"; // (e.g. utils.ts) can read it without importing main.ts and creating a circular // dependency (main → workspace → utils → main) that triggers a TDZ. // Re-exported from main.ts for backwards compatibility. -export const VERSION = "1.826.0"; +export const VERSION = "1.827.0"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e8ea3dece4..2f5ea5b721 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,12 +1,12 @@ { "name": "@windmill-labs/components", - "version": "1.826.0", + "version": "1.827.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@windmill-labs/components", - "version": "1.826.0", + "version": "1.827.0", "hasInstallScript": true, "license": "AGPL-3.0", "dependencies": { diff --git a/frontend/package.json b/frontend/package.json index 259a83b5da..4515459ba3 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "@windmill-labs/components", - "version": "1.826.0", + "version": "1.827.0", "scripts": { "dev": "vite dev", "dev:ui-builder": "mv static/ui_builder static/ui_builder.dev-disabled 2>/dev/null || true ; trap 'mv static/ui_builder.dev-disabled static/ui_builder 2>/dev/null || true' EXIT ; vite dev", diff --git a/frontend/src/lib/components/WorkerGroup.svelte b/frontend/src/lib/components/WorkerGroup.svelte index 65d00f9ba6..3b399a5a1e 100644 --- a/frontend/src/lib/components/WorkerGroup.svelte +++ b/frontend/src/lib/components/WorkerGroup.svelte @@ -104,6 +104,7 @@ autoscaling?: AutoscalingConfig native_mode?: boolean object_store_cache_config?: ObjectStoreConfig + paused?: boolean } = $state({}) function loadNConfig() { @@ -111,10 +112,9 @@ ? config.worker_tags != undefined || config.dedicated_worker != undefined || config.dedicated_workers != undefined - ? config - : { - worker_tags: [] - } + ? // a copy: editing the draft must not touch the saved config it is compared against + $state.snapshot(config) + : { ...$state.snapshot(config), worker_tags: [] } : { worker_tags: [] } @@ -212,6 +212,7 @@ periodic_script_interval_seconds?: number native_mode?: boolean object_store_cache_config?: ObjectStoreConfig + paused?: boolean } activeWorkers: number customTags: string[] | undefined @@ -268,8 +269,10 @@ if (!config && !nconfig) return false if (!config || !nconfig) return true - const cleaned1 = cleanValueProperties(config) - const cleaned2 = cleanValueProperties(nconfig) + // Compared without empty values: the draft is filled with empty defaults a saved + // config may lack, which are not an edit. + const cleaned1 = cleanWorkerGroupConfig(cleanValueProperties(config)) + const cleaned2 = cleanWorkerGroupConfig(cleanValueProperties(nconfig)) return ( orderedJsonStringify(replaceFalseWithUndefined(cleaned1)) !== @@ -298,6 +301,47 @@ let openDelete = $state(false) let openClean = $state(false) let openRestart = $state(false) + let pendingConfig: typeof nconfig | undefined = $state() + + // Mirrors `reload_worker_config` in backend/src/monitor.rs, which decides the restart. It + // compares the body that is sent rather than the draft, and only the fields a worker loads. + const RESTART_FIELDS = [ + 'dedicated_worker', + 'dedicated_workers', + 'init_bash', + 'periodic_script_bash', + 'periodic_script_interval_seconds', + 'native_mode' + ] as const + const OTHER_WORKER_FIELDS = [ + 'worker_tags', + 'priority_tags', + 'additional_python_paths', + 'pip_local_dependencies', + 'object_store_cache_config' + ] as const + function restartsWorkers(body: typeof nconfig): boolean { + if (activeWorkers === 0) return false + const saved: typeof nconfig = config ?? {} + const comparable = (c: typeof nconfig, k: keyof typeof nconfig) => + orderedJsonStringify(replaceFalseWithUndefined(cleanWorkerGroupConfig({ v: c[k] }))) + const changed = (k: keyof typeof nconfig) => comparable(saved, k) !== comparable(body, k) + const restartFieldChanged = RESTART_FIELDS.some(changed) + const onlyPauseChanged = + changed('paused') && + !restartFieldChanged && + !OTHER_WORKER_FIELDS.some(changed) && + !hasEnvVarChanges + const dedicated = + body.dedicated_worker != undefined || (body.dedicated_workers?.length ?? 0) > 0 + return !onlyPauseChanged && (dedicated || restartFieldChanged) + } + + async function saveConfig(body: typeof nconfig) { + await ConfigService.updateConfig({ name: 'worker__' + name, requestBody: body }) + sendUserToast('Configuration set') + dispatch('reload') + } // Compute hashed tags for display (actual tags used by the worker) let hashedDedicatedTags: Map = $state(new Map()) @@ -399,6 +443,31 @@ + { + pendingConfig = undefined + }} + on:confirmed={async () => { + const body = pendingConfig + pendingConfig = undefined + if (body) { + await saveConfig(body) + } + }} +> +
+ Applying these changes will restart the {pluralize(activeWorkers, 'worker')} of worker group '{name}'. + Workers finish their running job, shut down gracefully and are expected to be restarted by + their supervisor. + Pausing or resuming a worker group alone does not restart its workers. +
+ + {/if} + {#if nconfig !== undefined} +
+ + {/if} +