starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-09-21 00:02:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: agent workers v2 using http (#5588)

Browse Source
This commit is contained in:
Ruben Fiszel
2025-04-10 00:56:11 +00:00
committed by GitHub
parent eabd3d1346
commit 63fa499015
83 changed files with 4852 additions and 2719 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/docker-image.yml
+2 -2
View File
@@ -95,7 +95,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
features=embedding,parquet,openidconnect,jemalloc,license,http_trigger,zip,oauth2,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,agent_worker_server,all_languages
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.DEV_SHA }}
${{ steps.meta-public.outputs.tags }}
@@ -157,7 +157,7 @@ jobs:
platforms: linux/amd64,linux/arm64
push: true
build-args: |
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
features=enterprise,enterprise_saml,stripe,embedding,parquet,prometheus,openidconnect,cloud,jemalloc,agent_worker_server,tantivy,license,http_trigger,zip,oauth2,kafka,sqs_trigger,nats,otel,dind,postgres_trigger,mqtt_trigger,websocket,smtp,static_frontend,all_languages
tags: |
${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-ee:${{ env.DEV_SHA }}
${{ steps.meta-ee-public.outputs.tags }}
backend/.sqlx/query-0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET \n ping_at = now(), \n jobs_executed = 1, \n current_job_id = $1, \n current_job_workspace_id = 'admins' \n WHERE worker = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": []
},
"hash": "0e52a588f3edeb8fb58d6d62247b8590e51171e2811c62737bdb81fb0ac8f182"
}
backend/.sqlx/query-2ebb0463b790ddf7ba0ee22d8c9afc88eb57c4110a202775003fb48b2f4e317f.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'started_at'],\n to_jsonb(now()::text)\n )\n WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "2ebb0463b790ddf7ba0ee22d8c9afc88eb57c4110a202775003fb48b2f4e317f"
}
backend/.sqlx/query-6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21.json → backend/.sqlx/query-2faa27519624249f16cf89814ab5efe8f8daf928c1194cecacfa8223165fb9f2.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_runtime r SET\n memory_peak = $1,\n ping = now()\n FROM v2_job_queue q\n WHERE r.id = $2 AND q.id = r.id\n RETURNING canceled_by, canceled_reason",
"query": "UPDATE v2_job_runtime r SET\n memory_peak = $1,\n ping = now()\n FROM v2_job_queue q\n WHERE r.id = $2 AND q.id = r.id\n RETURNING canceled_by, canceled_reason",
"describe": {
"columns": [
{
@@ -25,5 +25,5 @@
true
]
},
"hash": "6ff7a025f529c077c1b6c9632a367aa29e2f0fdac3f1984550484d5a06a6ea21"
"hash": "2faa27519624249f16cf89814ab5efe8f8daf928c1194cecacfa8223165fb9f2"
}
backend/.sqlx/query-45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_status SET\n workflow_as_code_status = jsonb_set(\n jsonb_set(\n COALESCE(workflow_as_code_status, '{}'::jsonb),\n array[$1],\n COALESCE(workflow_as_code_status->$1, '{}'::jsonb)\n ),\n array[$1, 'started_at'],\n to_jsonb(now()::text)\n )\n WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "45d616c92ebcbe30a563e1fa7d2d0e53392e238144b039cfe042587d7fe1dea3"
}
backend/.sqlx/query-505250098ab003ff0ca30046df283e54bf44be74305070f10a5720a04c4789f3.json
Generated
+16
View File
@@ -0,0 +1,16 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET canceled_by = $1\n , canceled_reason = $2\nWHERE id = $3",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "505250098ab003ff0ca30046df283e54bf44be74305070f10a5720a04c4789f3"
}
backend/.sqlx/query-e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c.json → backend/.sqlx/query-506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
"query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
"describe": {
"columns": [],
"parameters": {
@@ -18,5 +18,5 @@
},
"nullable": []
},
"hash": "e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c"
"hash": "506066203c49424e9944eb3948dc1657d3d796e6233e9f0ec925879c705d4773"
}
backend/.sqlx/query-77701b16ee1f6dd827372835db59bbffc7254af47a8d48b7ba3cf969c2f8398c.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue SET tag = $1, running = false WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Varchar",
"Uuid"
]
},
"nullable": []
},
"hash": "77701b16ee1f6dd827372835db59bbffc7254af47a8d48b7ba3cf969c2f8398c"
}
backend/.sqlx/query-8efd06387ded837d7849adafe5bc93acb882ef90fc58b023650c875e0fd17047.json → backend/.sqlx/query-7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n values ($1, $2, $3, $4, $5, $6, $7, $8) \n ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8",
"query": "INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id) \n values ($1, $2, $3, $4, $5, $6, $7, $8) \n ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8",
"describe": {
"columns": [],
"parameters": {
@@ -17,5 +17,5 @@
},
"nullable": []
},
"hash": "8efd06387ded837d7849adafe5bc93acb882ef90fc58b023650c875e0fd17047"
"hash": "7d78efab0a588f56a13a7b5251f0a72f5d341b053218e7aec83a834cf7ccc98f"
}
backend/.sqlx/query-a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text) RETURNING length(logs)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "length",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Text",
"Uuid",
"Varchar"
]
},
"nullable": [
null
]
},
"hash": "a2e86f169ffbf8acee5f7c7b71db5859ac94ffbad267c9cd6c652e8ce8fc5d3c"
}
backend/.sqlx/query-03ae5b1c912b13a8a7aadf50cb4984a2ea952e782fd52eb3088454690bd13dd1.json → backend/.sqlx/query-a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar FROM script WHERE hash = $1 LIMIT 1",
"query": "SELECT content AS \"content!: String\",\n lock AS \"lock: String\", language AS \"language: Option<ScriptLang>\", envs AS \"envs: Vec<String>\", schema AS \"schema: String\", schema_validation AS \"schema_validation: bool\", codebase LIKE '%.tar' as use_tar FROM script WHERE hash = $1 LIMIT 1",
"describe": {
"columns": [
{
@@ -83,5 +83,5 @@
null
]
},
"hash": "03ae5b1c912b13a8a7aadf50cb4984a2ea952e782fd52eb3088454690bd13dd1"
"hash": "a9db7b2f435bb82acb8c5eeb7f800b28f3256491fdaa168591adc7b4b9f3327a"
}
backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json → backend/.sqlx/query-aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
"describe": {
"columns": [],
"parameters": {
@@ -20,5 +20,5 @@
},
"nullable": []
},
"hash": "a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb"
"hash": "aa523c363186575b4bd2537b8e2430e6938e7cc35f8c9e2d1c5459a85443cbdd"
}
backend/.sqlx/query-c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda.json
Generated
-15
View File
@@ -1,15 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE v2_job_queue\n SET canceled_by = 'timeout'\n , canceled_reason = $1\n WHERE id = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Uuid"
]
},
"nullable": []
},
"hash": "c00bae0d8c9bee37cbad4de4cb02c80d00f52a3fc32bf32271ebc90f7837abda"
}
backend/.sqlx/query-c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa.json
Generated
+15
View File
@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET \nping_at = now(), \njobs_executed = 1, \ncurrent_job_id = $1, \ncurrent_job_workspace_id = 'admins' \nWHERE worker = $2",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid",
"Text"
]
},
"nullable": []
},
"hash": "c3025cdb6e421e1225d420e8b1efd18d1dd3bb2fac53c1f2df648b61fb7488aa"
}
backend/.sqlx/query-f367a1c8f80dd414dcbcd949374eeb5770796f00b5b3d547163bcfdaed65d8ae.json → backend/.sqlx/query-c53e1c7133c8ae187656eef5999509fae17fb0ba43e084327accbb5b24c3dfbd.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO outstanding_wait_time(job_id, self_wait_time_ms) VALUES ($1, $2)\n ON CONFLICT (job_id) DO UPDATE SET self_wait_time_ms = EXCLUDED.self_wait_time_ms",
"query": "INSERT INTO outstanding_wait_time(job_id, self_wait_time_ms) VALUES ($1, $2)\n ON CONFLICT (job_id) DO UPDATE SET self_wait_time_ms = EXCLUDED.self_wait_time_ms",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "f367a1c8f80dd414dcbcd949374eeb5770796f00b5b3d547163bcfdaed65d8ae"
"hash": "c53e1c7133c8ae187656eef5999509fae17fb0ba43e084327accbb5b24c3dfbd"
}
backend/.sqlx/query-faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a.json
Generated
+22
View File
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT tag FROM v2_job WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "tag",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": [
false
]
},
"hash": "faf2c77242e0ab39b33886edf3b742531bf1351d0be1c3631bde0adfe375497a"
}
backend/.sqlx/query-fec6d5674dc6b5a6a0ece419c40508835affcb7679a48f2a443777e829bd1e74.json
Generated
+5 -5
View File
@@ -41,11 +41,11 @@
]
},
"nullable": [
true,
true,
true,
true,
true,
false,
false,
false,
false,
false,
true
]
},
backend/Cargo.lock
Generated
+122 -22
View File
@@ -2572,7 +2572,7 @@ dependencies = [
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core",
"parking_lot_core 0.9.10",
]
[[package]]
@@ -2626,7 +2626,7 @@ dependencies = [
"log",
"num_cpus",
"object_store",
"parking_lot",
"parking_lot 0.12.3",
"parquet",
"paste",
"pin-project-lite",
@@ -2687,7 +2687,7 @@ dependencies = [
"hashbrown 0.14.5",
"log",
"object_store",
"parking_lot",
"parking_lot 0.12.3",
"rand 0.8.5",
"tempfile",
"url",
@@ -2867,7 +2867,7 @@ dependencies = [
"itertools 0.12.1",
"log",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"pin-project-lite",
"rand 0.8.5",
"tokio",
@@ -2998,7 +2998,7 @@ dependencies = [
"indexmap 2.8.0",
"log",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"serde",
"serde_json",
"sha2 0.10.8",
@@ -3080,7 +3080,7 @@ dependencies = [
"indexmap 2.8.0",
"libc",
"memoffset",
"parking_lot",
"parking_lot 0.12.3",
"percent-encoding",
"pin-project",
"serde",
@@ -3325,7 +3325,7 @@ dependencies = [
"log",
"once_cell",
"os_pipe",
"parking_lot",
"parking_lot 0.12.3",
"pin-project",
"rand 0.8.5",
"tokio",
@@ -3705,7 +3705,7 @@ dependencies = [
"log",
"node_resolver",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"sys_traits",
"thiserror 2.0.12",
"url",
@@ -3862,7 +3862,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d774fd83f26b24f0805a6ab8b26834a0d06ceac0db517b769b1e4633c96a2057"
dependencies = [
"futures",
"parking_lot",
"parking_lot 0.12.3",
"tokio",
]
@@ -4972,7 +4972,7 @@ checksum = "1d930c203dd0b6ff06e0201a4a2fe9149b43c684fd4420555b26d21b1a02956f"
dependencies = [
"futures-core",
"lock_api",
"parking_lot",
"parking_lot 0.12.3",
]
[[package]]
@@ -5638,7 +5638,7 @@ dependencies = [
"ipconfig",
"moka",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"rand 0.9.0",
"resolv-conf",
"serde",
@@ -7230,7 +7230,7 @@ dependencies = [
"crossbeam-epoch",
"crossbeam-utils",
"loom",
"parking_lot",
"parking_lot 0.12.3",
"portable-atomic",
"rustc_version 0.4.1",
"smallvec",
@@ -7933,7 +7933,7 @@ dependencies = [
"hyper 1.6.0",
"itertools 0.13.0",
"md-5 0.10.6",
"parking_lot",
"parking_lot 0.12.3",
"percent-encoding",
"quick-xml 0.36.2",
"rand 0.8.5",
@@ -8349,6 +8349,17 @@ version = "2.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
[[package]]
name = "parking_lot"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
dependencies = [
"instant",
"lock_api",
"parking_lot_core 0.8.6",
]
[[package]]
name = "parking_lot"
version = "0.12.3"
@@ -8356,7 +8367,21 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
dependencies = [
"lock_api",
"parking_lot_core",
"parking_lot_core 0.9.10",
]
[[package]]
name = "parking_lot_core"
version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a2cfe6f0ad2bfc16aefa463b497d5c7a5ecd44a23efa72aa342d90177356dc"
dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall 0.2.16",
"smallvec",
"winapi",
]
[[package]]
@@ -8957,7 +8982,7 @@ dependencies = [
"fnv",
"lazy_static",
"memchr",
"parking_lot",
"parking_lot 0.12.3",
"thiserror 2.0.12",
]
@@ -9111,7 +9136,7 @@ dependencies = [
"ahash 0.8.11",
"equivalent",
"hashbrown 0.15.2",
"parking_lot",
"parking_lot 0.12.3",
]
[[package]]
@@ -9359,6 +9384,15 @@ version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430"
[[package]]
name = "redox_syscall"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.3.5"
@@ -9568,6 +9602,43 @@ dependencies = [
"windows-registry",
]
[[package]]
name = "reqwest-middleware"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64e8975513bd9a7a43aad01030e79b3498e05db14e9d945df6483e8cf9b8c4c4"
dependencies = [
"anyhow",
"async-trait",
"http 1.3.1",
"reqwest 0.12.15",
"serde",
"thiserror 1.0.69",
"tower-service",
]
[[package]]
name = "reqwest-retry"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "29c73e4195a6bfbcb174b790d9b3407ab90646976c55de58a6515da25d851178"
dependencies = [
"anyhow",
"async-trait",
"futures",
"getrandom 0.2.15",
"http 1.3.1",
"hyper 1.6.0",
"parking_lot 0.11.2",
"reqwest 0.12.15",
"reqwest-middleware",
"retry-policies",
"thiserror 1.0.69",
"tokio",
"tracing",
"wasm-timer",
]
[[package]]
name = "resolv-conf"
version = "0.7.1"
@@ -9577,6 +9648,15 @@ dependencies = [
"hostname",
]
[[package]]
name = "retry-policies"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5875471e6cab2871bc150ecb8c727db5113c9338cc3354dc5ee3425b6aa40a1c"
dependencies = [
"rand 0.8.5",
]
[[package]]
name = "rfc6979"
version = "0.4.0"
@@ -12264,7 +12344,7 @@ dependencies = [
"bytes",
"libc",
"mio 1.0.3",
"parking_lot",
"parking_lot 0.12.3",
"pin-project-lite",
"signal-hook-registry",
"socket2",
@@ -12328,7 +12408,7 @@ dependencies = [
"futures-channel",
"futures-util",
"log",
"parking_lot",
"parking_lot 0.12.3",
"percent-encoding",
"phf",
"pin-project-lite",
@@ -12354,7 +12434,7 @@ dependencies = [
"futures-channel",
"futures-util",
"log",
"parking_lot",
"parking_lot 0.12.3",
"percent-encoding",
"phf",
"pin-project-lite",
@@ -12613,7 +12693,7 @@ dependencies = [
"cookie 0.18.1",
"futures-util",
"http 1.3.1",
"parking_lot",
"parking_lot 0.12.3",
"pin-project-lite",
"tower-layer",
"tower-service",
@@ -13460,6 +13540,21 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wasm-timer"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be0ecb0db480561e9a7642b5d3e4187c128914e58aa84330b9493e3eb68c5e7f"
dependencies = [
"futures",
"js-sys",
"parking_lot 0.11.2",
"pin-utils",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "wasm_dep_analyzer"
version = "0.2.0"
@@ -13533,7 +13628,7 @@ dependencies = [
"log",
"naga",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"profiling",
"raw-window-handle",
"ron",
@@ -13575,7 +13670,7 @@ dependencies = [
"ndk-sys",
"objc",
"once_cell",
"parking_lot",
"parking_lot 0.12.3",
"profiling",
"range-alloc",
"raw-window-handle",
@@ -13813,8 +13908,10 @@ dependencies = [
"windmill-indexer",
"windmill-parser",
"windmill-parser-py",
"windmill-parser-py-imports",
"windmill-parser-ts",
"windmill-queue",
"windmill-worker",
]
[[package]]
@@ -13901,6 +13998,8 @@ dependencies = [
"rand 0.9.0",
"regex",
"reqwest 0.12.15",
"reqwest-middleware",
"reqwest-retry",
"semver 1.0.26",
"serde",
"serde_json",
@@ -14267,6 +14366,7 @@ dependencies = [
"rand 0.9.0",
"regex",
"reqwest 0.12.15",
"reqwest-middleware",
"rust_decimal",
"serde",
"serde_json",
backend/Cargo.toml
+4
View File
@@ -49,6 +49,7 @@ lto = "thin"
[features]
default = []
agent_worker_server = ["windmill-api/agent_worker_server"]
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "dep:windmill-autoscaling", "windmill-autoscaling/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise"]
enterprise_saml = ["windmill-api/enterprise_saml", "oauth2"]
stripe = ["windmill-api/stripe"]
@@ -170,6 +171,9 @@ windmill-parser-graphql = { path = "./parsers/windmill-parser-graphql" }
windmill-parser-php = { path = "./parsers/windmill-parser-php" }
windmill-api-client = { path = "./windmill-api-client" }
reqwest-retry = "^0"
reqwest-middleware = { version = "^0", features = ["json"] }
rustls = "0.23.0"
memchr = "2.7.4"
axum = { version = "^0.7", features = ["multipart"] }
backend/ee-repo-ref.txt
+1 -1
View File
@@ -1 +1 @@
619f7dcd1097bfcd2f8b85f2075782219ff76be1
b769ca52ebdaf2c1ad110622e258be06d00eb164
backend/src/main.rs
+468 -393
View File
File diff suppressed because it is too large Load Diff
backend/src/monitor.rs
+213 -148
View File
@@ -13,7 +13,7 @@ use std::{
use chrono::{NaiveDateTime, Utc};
use futures::{stream::FuturesUnordered, StreamExt};
use serde::{de::DeserializeOwned, Deserializer};
use serde::{de::DeserializeOwned, Deserializer};
use sqlx::{Pool, Postgres};
use tokio::{
join,
@@ -34,7 +34,7 @@ use windmill_common::ee::{jobs_waiting_alerts, worker_groups_alerts};
#[cfg(feature = "oauth2")]
use windmill_common::global_settings::OAUTH_SETTING;
use windmill_common::{
ee::CriticalErrorChannel, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{
agent_workers::DECODED_AGENT_TOKEN, auth::create_token_for_owner, ee::CriticalErrorChannel, error, flow_status::{FlowStatus, FlowStatusModule}, global_settings::{
BASE_URL_SETTING, BUNFIG_INSTALL_SCOPES_SETTING, CRITICAL_ALERT_MUTE_UI_SETTING,
CRITICAL_ERROR_CHANNELS_SETTING, DEFAULT_TAGS_PER_WORKSPACE_SETTING,
DEFAULT_TAGS_WORKSPACES_SETTING, EXPOSE_DEBUG_METRICS_SETTING, EXPOSE_METRICS_SETTING,
@@ -45,14 +45,12 @@ use windmill_common::{
REQUIRE_PREEXISTING_USER_FOR_OAUTH_SETTING, RETENTION_PERIOD_SECS_SETTING,
SAML_METADATA_SETTING, SCIM_TOKEN_SETTING, TIMEOUT_WAIT_RESULT_SETTING,
}, indexer::load_indexer_config, jobs::QueuedJob, jwt::JWT_SECRET, oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH, server::load_smtp_config, tracing_init::JSON_FMT, users::truncate_token, utils::{now_from_db, rd_string, report_critical_error, Mode}, worker::{
load_worker_config, make_pull_query, make_suspended_pull_query, reload_custom_tags_setting,
update_min_version, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG,
SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP,
load_worker_config, reload_custom_tags_setting, store_pull_query, store_suspended_pull_query, update_min_version, Connection, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, INDEXER_CONFIG, SCRIPT_TOKEN_EXPIRY, SMTP_CONFIG, TMP_DIR, WORKER_CONFIG, WORKER_GROUP
}, KillpillSender, BASE_URL, CRITICAL_ALERT_MUTE_UI_ENABLED, CRITICAL_ERROR_CHANNELS, DB, DEFAULT_HUB_BASE_URL, HUB_BASE_URL, JOB_RETENTION_SECS, METRICS_DEBUG_ENABLED, METRICS_ENABLED, MONITOR_LOGS_ON_OBJECT_STORE, OTEL_LOGS_ENABLED, OTEL_METRICS_ENABLED, OTEL_TRACING_ENABLED, SERVICE_LOG_RETENTION_SECS
};
use windmill_queue::{cancel_job, MiniPulledJob};
use windmill_queue::{cancel_job, MiniPulledJob, SameWorkerPayload};
use windmill_worker::{
create_token_for_owner, handle_job_error, AuthedClient, SameWorkerPayload, SameWorkerSender, SendResult, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, SCRIPT_TOKEN_EXPIRY
handle_job_error, AuthedClient, JobCompletedSender, SameWorkerSender, BUNFIG_INSTALL_SCOPES, INSTANCE_PYTHON_VERSION, JOB_DEFAULT_TIMEOUT, KEEP_JOB_DIR, MAVEN_REPOS, NO_DEFAULT_MAVEN, NPM_CONFIG_REGISTRY, NUGET_CONFIG, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL
};
#[cfg(feature = "parquet")]
@@ -113,88 +111,109 @@ lazy_static::lazy_static! {
.unwrap_or(false);
static ref QUEUE_COUNT_TAGS: Arc<RwLock<Vec<String>>> = Arc::new(RwLock::new(Vec::new()));
static ref DISABLE_CONCURRENCY_LIMIT: bool = std::env::var("DISABLE_CONCURRENCY_LIMIT").is_ok_and(|s| s == "true");
}
pub async fn initial_load(
db: &Pool<Postgres>,
conn: &Connection,
tx: KillpillSender,
worker_mode: bool,
server_mode: bool,
#[cfg(feature = "parquet")] disable_s3_store: bool,
) {
if let Err(e) = load_metrics_enabled(db).await {
if let Err(e) = load_metrics_enabled(conn).await {
tracing::error!("Error loading expose metrics: {e:#}");
}
if let Err(e) = load_metrics_debug_enabled(db).await {
if let Err(e) = load_metrics_debug_enabled(conn).await {
tracing::error!("Error loading expose debug metrics: {e:#}");
}
if let Err(e) = reload_critical_alert_mute_ui_setting(db).await {
if let Err(e) = reload_critical_alert_mute_ui_setting(conn).await {
tracing::error!("Error loading critical alert mute ui setting: {e:#}");
}
if let Err(e) = load_tag_per_workspace_enabled(db).await {
tracing::error!("Error loading default tag per workpsace: {e:#}");
}
if let Some(db) = conn.as_sql() {
if let Err(e) = load_tag_per_workspace_enabled(db).await {
tracing::error!("Error loading default tag per workpsace: {e:#}");
}
if let Err(e) = load_tag_per_workspace_workspaces(db).await {
tracing::error!("Error loading default tag per workpsace workspaces: {e:#}");
if let Err(e) = load_tag_per_workspace_workspaces(db).await {
tracing::error!("Error loading default tag per workpsace workspaces: {e:#}");
}
}
if server_mode {
load_require_preexisting_user(db).await;
if let Some(db) = conn.as_sql() {
load_require_preexisting_user(db).await;
}
}
if worker_mode {
load_keep_job_dir(db).await;
reload_worker_config(&db, tx, false).await;
load_keep_job_dir(conn).await;
match conn {
Connection::Sql(db) => {
reload_worker_config(&db, tx, false).await;
}
Connection::Http(_) => {
// TODO: reload worker config from http
WORKER_CONFIG.write().await.worker_tags = DECODED_AGENT_TOKEN.as_ref().map(|x| x.tags.clone()).unwrap_or_default();
}
}
}
if let Err(e) = reload_custom_tags_setting(db).await {
tracing::error!("Error reloading custom tags: {:?}", e)
}
if let Err(e) = reload_hub_base_url_setting(db, server_mode).await {
if let Err(e) = reload_hub_base_url_setting(conn, server_mode).await {
tracing::error!("Error reloading hub base url: {:?}", e)
}
if let Err(e) = reload_jwt_secret_setting(&db).await {
tracing::error!("Could not reload jwt secret setting: {:?}", e);
if let Some(db) = conn.as_sql() {
if let Err(e) = reload_jwt_secret_setting(db).await {
tracing::error!("Could not reload jwt secret setting: {:?}", e);
}
if let Err(e) = reload_custom_tags_setting(db).await {
tracing::error!("Error reloading custom tags: {:?}", e)
}
}
#[cfg(feature = "parquet")]
if !disable_s3_store {
reload_s3_cache_setting(&db).await;
if let Some(db) = conn.as_sql() {
reload_s3_cache_setting(db).await;
}
}
reload_smtp_config(&db).await;
if let Some(db) = conn.as_sql() {
reload_smtp_config(db).await;
}
if server_mode {
reload_retention_period_setting(&db).await;
reload_request_size(&db).await;
reload_saml_metadata_setting(&db).await;
reload_scim_token_setting(&db).await;
reload_retention_period_setting(&conn).await;
reload_request_size(&conn).await;
reload_saml_metadata_setting(&conn).await;
reload_scim_token_setting(&conn).await;
}
if worker_mode {
reload_job_default_timeout_setting(&db).await;
reload_extra_pip_index_url_setting(&db).await;
reload_pip_index_url_setting(&db).await;
reload_npm_config_registry_setting(&db).await;
reload_bunfig_install_scopes_setting(&db).await;
reload_instance_python_version_setting(&db).await;
reload_nuget_config_setting(&db).await;
reload_maven_repos_setting(&db).await;
reload_no_default_maven_setting(&db).await;
reload_job_default_timeout_setting(&conn).await;
reload_extra_pip_index_url_setting(&conn).await;
reload_pip_index_url_setting(&conn).await;
reload_npm_config_registry_setting(&conn).await;
reload_bunfig_install_scopes_setting(&conn).await;
reload_instance_python_version_setting(&conn).await;
reload_nuget_config_setting(&conn).await;
reload_maven_repos_setting(&conn).await;
reload_no_default_maven_setting(&conn).await;
}
}
pub async fn load_metrics_enabled(db: &DB) -> error::Result<()> {
let metrics_enabled = load_value_from_global_settings(db, EXPOSE_METRICS_SETTING).await;
pub async fn load_metrics_enabled(conn: &Connection) -> error::Result<()> {
let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_METRICS_SETTING, true).await;
match metrics_enabled {
Ok(Some(serde_json::Value::Bool(t))) => METRICS_ENABLED.store(t, Ordering::Relaxed),
_ => (),
@@ -317,9 +336,9 @@ pub async fn load_tag_per_workspace_workspaces(db: &DB) -> error::Result<()> {
Ok(())
}
pub async fn reload_critical_alert_mute_ui_setting(db: &DB) -> error::Result<()> {
pub async fn reload_critical_alert_mute_ui_setting(conn: &Connection) -> error::Result<()> {
if let Ok(Some(serde_json::Value::Bool(t))) =
load_value_from_global_settings(db, CRITICAL_ALERT_MUTE_UI_SETTING).await
load_value_from_global_settings_with_conn(conn, CRITICAL_ALERT_MUTE_UI_SETTING, true).await
{
CRITICAL_ALERT_MUTE_UI_ENABLED.store(t, Ordering::Relaxed);
@@ -327,8 +346,8 @@ pub async fn reload_critical_alert_mute_ui_setting(db: &DB) -> error::Result<()>
Ok(())
}
pub async fn load_metrics_debug_enabled(db: &DB) -> error::Result<()> {
let metrics_enabled = load_value_from_global_settings(db, EXPOSE_DEBUG_METRICS_SETTING).await;
pub async fn load_metrics_debug_enabled(conn: &Connection) -> error::Result<()> {
let metrics_enabled = load_value_from_global_settings_with_conn(conn, EXPOSE_DEBUG_METRICS_SETTING, true).await;
match metrics_enabled {
Ok(Some(serde_json::Value::Bool(t))) => {
METRICS_DEBUG_ENABLED.store(t, Ordering::Relaxed);
@@ -479,8 +498,8 @@ fn get_worker_group(mode: &Mode) -> Option<String> {
}
}
pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
let db = db.clone();
pub fn send_logs_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) {
let conn = conn.clone();
let hostname = hostname.to_string();
let mode = mode.clone();
let worker_group = get_worker_group(&mode);
@@ -495,7 +514,7 @@ pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
&hostname,
&mode,
&worker_group,
&db,
&conn,
snd_highest_file,
false,
)
@@ -504,11 +523,11 @@ pub fn send_logs_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
});
}
pub async fn send_current_log_file_to_object_store(db: &DB, hostname: &str, mode: &Mode) {
pub async fn send_current_log_file_to_object_store(conn: &Connection, hostname: &str, mode: &Mode) {
tracing::info!("Sending current log file to object store");
let (highest_file, _) = find_two_highest_files(hostname).await;
let worker_group = get_worker_group(&mode);
send_log_file_to_object_store(hostname, mode, &worker_group, db, highest_file, true).await;
send_log_file_to_object_store(hostname, mode, &worker_group, conn, highest_file, true).await;
}
fn get_now_and_str() -> (NaiveDateTime, String) {
@@ -528,7 +547,7 @@ async fn send_log_file_to_object_store(
hostname: &str,
mode: &Mode,
worker_group: &Option<String>,
db: &Pool<Postgres>,
conn: &Connection,
snd_highest_file: Option<String>,
use_now: bool,
) {
@@ -588,18 +607,23 @@ async fn send_log_file_to_object_store(
let (ok_lines, err_lines) = read_log_counters(ts_str);
if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)",
hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT)
.execute(db)
.await {
tracing::error!("Error inserting log file: {:?}", e);
} else {
if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| {
last_log_file_sent.replace(ts);
}) {
tracing::error!("Error updating last log file sent: {:?}", e);
if let Some(db) = conn.as_sql() {
if let Err(e) = sqlx::query!("INSERT INTO log_file (hostname, mode, worker_group, log_ts, file_path, ok_lines, err_lines, json_fmt) VALUES ($1, $2::text::LOG_MODE, $3, $4, $5, $6, $7, $8)",
hostname, mode.to_string(), worker_group.clone(), ts, highest_file, ok_lines as i64, err_lines as i64, *JSON_FMT)
.execute(db)
.await {
tracing::error!("Error inserting log file: {:?}", e);
} else {
if let Err(e) = LAST_LOG_FILE_SENT.lock().map(|mut last_log_file_sent| {
last_log_file_sent.replace(ts);
}) {
tracing::error!("Error updating last log file sent: {:?}", e);
}
tracing::info!("Log file sent: {}", highest_file);
}
tracing::info!("Log file sent: {}", highest_file);
} else {
// tracing::warn!("Not sending log file to object store in agent mode");
()
}
}
}
@@ -622,8 +646,8 @@ fn read_log_counters(ts_str: String) -> (usize, usize) {
(ok_lines, err_lines)
}
pub async fn load_keep_job_dir(db: &DB) {
let value = load_value_from_global_settings(db, KEEP_JOB_DIR_SETTING).await;
pub async fn load_keep_job_dir(conn: &Connection) {
let value = load_value_from_global_settings_with_conn(conn, KEEP_JOB_DIR_SETTING, true).await;
match value {
Ok(Some(serde_json::Value::Bool(t))) => KEEP_JOB_DIR.store(t, Ordering::Relaxed),
Err(e) => {
@@ -890,23 +914,23 @@ async fn delete_log_files_from_disk_and_store(
let _: Vec<_> = delete_futures.collect().await;
}
pub async fn reload_scim_token_setting(db: &DB) {
reload_option_setting_with_tracing(db, SCIM_TOKEN_SETTING, "SCIM_TOKEN", SCIM_TOKEN.clone())
pub async fn reload_scim_token_setting(conn: &Connection) {
reload_option_setting_with_tracing(conn, SCIM_TOKEN_SETTING, "SCIM_TOKEN", SCIM_TOKEN.clone())
.await;
}
pub async fn reload_timeout_wait_result_setting(db: &DB) {
pub async fn reload_timeout_wait_result_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
TIMEOUT_WAIT_RESULT_SETTING,
"TIMEOUT_WAIT_RESULT",
TIMEOUT_WAIT_RESULT.clone(),
)
.await;
}
pub async fn reload_saml_metadata_setting(db: &DB) {
pub async fn reload_saml_metadata_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
SAML_METADATA_SETTING,
"SAML_METADATA",
SAML_METADATA.clone(),
@@ -914,9 +938,9 @@ pub async fn reload_saml_metadata_setting(db: &DB) {
.await;
}
pub async fn reload_extra_pip_index_url_setting(db: &DB) {
pub async fn reload_extra_pip_index_url_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
EXTRA_PIP_INDEX_URL_SETTING,
"PIP_EXTRA_INDEX_URL",
PIP_EXTRA_INDEX_URL.clone(),
@@ -924,9 +948,9 @@ pub async fn reload_extra_pip_index_url_setting(db: &DB) {
.await;
}
pub async fn reload_pip_index_url_setting(db: &DB) {
pub async fn reload_pip_index_url_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
PIP_INDEX_URL_SETTING,
"PIP_INDEX_URL",
PIP_INDEX_URL.clone(),
@@ -934,9 +958,9 @@ pub async fn reload_pip_index_url_setting(db: &DB) {
.await;
}
pub async fn reload_instance_python_version_setting(db: &DB) {
pub async fn reload_instance_python_version_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
INSTANCE_PYTHON_VERSION_SETTING,
"INSTANCE_PYTHON_VERSION",
INSTANCE_PYTHON_VERSION.clone(),
@@ -944,9 +968,9 @@ pub async fn reload_instance_python_version_setting(db: &DB) {
.await;
}
pub async fn reload_npm_config_registry_setting(db: &DB) {
pub async fn reload_npm_config_registry_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
NPM_CONFIG_REGISTRY_SETTING,
"NPM_CONFIG_REGISTRY",
NPM_CONFIG_REGISTRY.clone(),
@@ -954,9 +978,9 @@ pub async fn reload_npm_config_registry_setting(db: &DB) {
.await;
}
pub async fn reload_bunfig_install_scopes_setting(db: &DB) {
pub async fn reload_bunfig_install_scopes_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
BUNFIG_INSTALL_SCOPES_SETTING,
"BUNFIG_INSTALL_SCOPES",
BUNFIG_INSTALL_SCOPES.clone(),
@@ -964,21 +988,21 @@ pub async fn reload_bunfig_install_scopes_setting(db: &DB) {
.await;
}
pub async fn reload_nuget_config_setting(db: &DB) {
pub async fn reload_nuget_config_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
NUGET_CONFIG_SETTING,
"NUGET_CONFIG",
NUGET_CONFIG.clone(),
)
.await;
}
pub async fn reload_maven_repos_setting(db: &DB) {
reload_option_setting_with_tracing(db, windmill_common::global_settings::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone())
pub async fn reload_maven_repos_setting(conn: &Connection) {
reload_option_setting_with_tracing(conn, windmill_common::global_settings::MAVEN_REPOS_SETTING, "MAVEN_REPOS", MAVEN_REPOS.clone())
.await;
}
pub async fn reload_no_default_maven_setting(db: &DB) {
let value = load_value_from_global_settings(db, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING).await;
pub async fn reload_no_default_maven_setting(conn: &Connection) {
let value = load_value_from_global_settings_with_conn(conn, windmill_common::global_settings::NO_DEFAULT_MAVEN_SETTING, true).await;
match value {
Ok(Some(serde_json::Value::Bool(t))) => NO_DEFAULT_MAVEN.store(t, Ordering::Relaxed),
Err(e) => {
@@ -988,9 +1012,9 @@ pub async fn reload_no_default_maven_setting(db: &DB) {
};
}
pub async fn reload_retention_period_setting(db: &DB) {
pub async fn reload_retention_period_setting(conn: &Connection) {
if let Err(e) = reload_setting(
db,
conn,
RETENTION_PERIOD_SECS_SETTING,
"JOB_RETENTION_SECS",
60 * 60 * 24 * 30,
@@ -1002,9 +1026,9 @@ pub async fn reload_retention_period_setting(db: &DB) {
tracing::error!("Error reloading retention period: {:?}", e)
}
}
pub async fn reload_delete_logs_periodically_setting(db: &DB) {
pub async fn reload_delete_logs_periodically_setting(conn: &Connection) {
if let Err(e) = reload_setting(
db,
conn,
MONITOR_LOGS_ON_OBJECT_STORE_SETTING,
"MONITOR_LOGS_ON_OBJECT_STORE",
false,
@@ -1072,9 +1096,9 @@ pub async fn reload_s3_cache_setting(db: &DB) {
}
}
pub async fn reload_job_default_timeout_setting(db: &DB) {
pub async fn reload_job_default_timeout_setting(conn: &Connection) {
reload_option_setting_with_tracing(
db,
conn,
JOB_DEFAULT_TIMEOUT_SECS_SETTING,
"JOB_DEFAULT_TIMEOUT_SECS",
JOB_DEFAULT_TIMEOUT.clone(),
@@ -1082,9 +1106,9 @@ pub async fn reload_job_default_timeout_setting(db: &DB) {
.await;
}
pub async fn reload_request_size(db: &DB) {
pub async fn reload_request_size(conn: &Connection) {
if let Err(e) = reload_setting(
db,
conn,
REQUEST_SIZE_LIMIT_SETTING,
"REQUEST_SIZE_LIMIT",
DEFAULT_BODY_LIMIT,
@@ -1097,8 +1121,8 @@ pub async fn reload_request_size(db: &DB) {
}
}
pub async fn reload_license_key(db: &DB) -> anyhow::Result<()> {
let q = load_value_from_global_settings(db, LICENSE_KEY_SETTING)
pub async fn reload_license_key(conn: &Connection) -> anyhow::Result<()> {
let q = load_value_from_global_settings_with_conn(conn, LICENSE_KEY_SETTING, true)
.await
.map_err(|err| anyhow::anyhow!("Error reloading license key: {}", err.to_string()))?;
@@ -1123,12 +1147,12 @@ pub async fn reload_license_key(db: &DB) -> anyhow::Result<()> {
}
pub async fn reload_option_setting_with_tracing<T: FromStr + DeserializeOwned>(
db: &DB,
conn: &Connection,
setting_name: &str,
std_env_var: &str,
lock: Arc<RwLock<Option<T>>>,
) {
if let Err(e) = reload_option_setting(db, setting_name, std_env_var, lock.clone()).await {
if let Err(e) = reload_option_setting(conn, setting_name, std_env_var, lock.clone()).await {
tracing::error!("Error reloading setting {}: {:?}", setting_name, e)
}
}
@@ -1147,8 +1171,28 @@ pub async fn load_value_from_global_settings(
Ok(r)
}
pub async fn load_value_from_global_settings_with_conn(
conn: &Connection,
setting_name: &str,
load_from_http: bool,
) -> anyhow::Result<Option<serde_json::Value>> {
match conn {
Connection::Sql(db) => Ok(load_value_from_global_settings(db, setting_name).await?),
Connection::Http(client) => {
if load_from_http {
client.get::<Option<serde_json::Value>>(&format!("/api/agent_workers/get_global_setting/{}", setting_name)).await
.map_err(|e| anyhow::anyhow!("Error loading setting {}: {}", setting_name, e))
} else {
Ok(None)
}
}
}
}
pub async fn reload_option_setting<T: FromStr + DeserializeOwned>(
db: &DB,
conn: &Connection,
setting_name: &str,
std_env_var: &str,
lock: Arc<RwLock<Option<T>>>,
@@ -1163,7 +1207,7 @@ pub async fn reload_option_setting<T: FromStr + DeserializeOwned>(
return Ok(());
}
let q = load_value_from_global_settings(db, setting_name).await?;
let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?;
let mut value = std::env::var(std_env_var)
.ok()
@@ -1190,14 +1234,14 @@ pub async fn reload_option_setting<T: FromStr + DeserializeOwned>(
}
pub async fn reload_setting<T: FromStr + DeserializeOwned + Display>(
db: &DB,
conn: &Connection,
setting_name: &str,
std_env_var: &str,
default: T,
lock: Arc<RwLock<T>>,
transformer: fn(T) -> T,
) -> error::Result<()> {
let q = load_value_from_global_settings(db, setting_name).await?;
let q = load_value_from_global_settings_with_conn(conn, setting_name, true).await?;
let mut value = std::env::var(std_env_var)
.ok()
@@ -1255,27 +1299,32 @@ pub async fn monitor_pool(db: &DB) {
}
pub async fn monitor_db(
db: &Pool<Postgres>,
conn: &Connection,
base_internal_url: &str,
server_mode: bool,
_worker_mode: bool,
initial_load: bool,
_killpill_tx: KillpillSender,
) {
tracing::info!("Starting periodic monitor task");
let zombie_jobs_f = async {
if server_mode && !initial_load && !*DISABLE_ZOMBIE_JOBS_MONITORING {
if let Some(db) = conn.as_sql() {
handle_zombie_jobs(db, base_internal_url, "server").await;
match handle_zombie_flows(db).await {
Err(err) => {
tracing::error!("Error handling zombie flows: {:?}", err);
}
},
_ => {}
}
}
}
};
let expired_items_f = async {
if server_mode && !initial_load {
if let Some(db) = conn.as_sql() {
delete_expired_items(&db).await;
}
}
};
@@ -1288,35 +1337,43 @@ pub async fn monitor_db(
let expose_queue_metrics_f = async {
if !initial_load && server_mode {
expose_queue_metrics(&db).await;
if let Some(db) = conn.as_sql() {
expose_queue_metrics(&db).await;
}
}
};
let worker_groups_alerts_f = async {
#[cfg(feature = "enterprise")]
if server_mode && !initial_load {
worker_groups_alerts(&db).await;
if let Some(db) = conn.as_sql() {
worker_groups_alerts(&db).await;
}
}
};
let jobs_waiting_alerts_f = async {
#[cfg(feature = "enterprise")]
if server_mode {
jobs_waiting_alerts(&db).await;
if let Some(db) = conn.as_sql() {
jobs_waiting_alerts(&db).await;
}
}
};
let apply_autoscaling_f = async {
#[cfg(feature = "enterprise")]
if server_mode && !initial_load {
if let Err(e) = windmill_autoscaling::apply_all_autoscaling(db).await {
tracing::error!("Error applying autoscaling: {:?}", e);
if let Some(db) = conn.as_sql() {
if let Err(e) = windmill_autoscaling::apply_all_autoscaling(db).await {
tracing::error!("Error applying autoscaling: {:?}", e);
}
}
}
};
let update_min_worker_version_f = async {
update_min_version(db).await;
update_min_version(conn).await;
};
join!(
@@ -1329,6 +1386,7 @@ pub async fn monitor_db(
apply_autoscaling_f,
update_min_worker_version_f,
);
tracing::info!("Periodic monitor task completed");
}
pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
@@ -1439,7 +1497,7 @@ pub async fn reload_worker_config(
tx: KillpillSender,
kill_if_change: bool,
) {
let config = load_worker_config(&db, tx.clone()).await;
let config = load_worker_config(db, tx.clone()).await;
if let Err(e) = config {
tracing::error!("Error reloading worker config: {:?}", e)
} else {
@@ -1473,15 +1531,15 @@ pub async fn reload_worker_config(
let mut wc = WORKER_CONFIG.write().await;
tracing::info!("Reloading worker config...");
make_suspended_pull_query(&config).await;
make_pull_query(&config).await;
store_suspended_pull_query(&config).await;
store_pull_query(&config).await;
*wc = config
}
}
}
pub async fn load_base_url(db: &DB) -> error::Result<String> {
let q_base_url = load_value_from_global_settings(db, BASE_URL_SETTING).await?;
pub async fn load_base_url(conn: &Connection) -> error::Result<String> {
let q_base_url = load_value_from_global_settings_with_conn(conn, BASE_URL_SETTING, false).await?;
let std_base_url = std::env::var("BASE_URL")
.ok()
@@ -1511,34 +1569,39 @@ pub async fn load_base_url(db: &DB) -> error::Result<String> {
Ok(base_url)
}
pub async fn reload_base_url_setting(db: &DB) -> error::Result<()> {
#[cfg(feature = "oauth2")]
let q_oauth = load_value_from_global_settings(db, OAUTH_SETTING).await?;
pub async fn reload_base_url_setting(conn: &Connection) -> error::Result<()> {
#[cfg(feature = "oauth2")]
let oauths = if let Some(q) = q_oauth {
if let Ok(v) = serde_json::from_value::<
Option<HashMap<String, windmill_api::oauth2_ee::OAuthClient>>,
>(q.clone())
{
v
let oauths = if let Some(db) = conn.as_sql() {
let q_oauth = load_value_from_global_settings (db, OAUTH_SETTING).await?;
if let Some(q) = q_oauth {
if let Ok(v) = serde_json::from_value::<
Option<HashMap<String, windmill_api::oauth2_ee::OAuthClient>>,
>(q.clone())
{
v
} else {
tracing::error!("Could not parse oauth setting as a json, found: {:#?}", &q);
None
}
} else {
tracing::error!("Could not parse oauth setting as a json, found: {:#?}", &q);
None
}
} else {
None
};
let base_url = load_base_url(db).await?;
let base_url = load_base_url(conn).await?;
let is_secure = base_url.starts_with("https://");
#[cfg(feature = "oauth2")]
{
let mut l = windmill_api::OAUTH_CLIENTS.write().await;
*l = windmill_api::oauth2_ee::build_oauth_clients(&base_url, oauths, db).await
.map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
.unwrap();
if let Some(db) = conn.as_sql() {
let mut l = windmill_api::OAUTH_CLIENTS.write().await;
*l = windmill_api::oauth2_ee::build_oauth_clients(&base_url, oauths, db).await
.map_err(|e| tracing::error!("Error building oauth clients (is the oauth.json mounted and in correct format? Use '{}' as minimal oauth.json): {}", "{}", e))
.unwrap();
}
}
{
@@ -1796,7 +1859,7 @@ async fn handle_zombie_jobs(db: &Pool<Postgres>, base_internal_url: &str, worker
mpsc::channel::<SameWorkerPayload>(1);
let same_worker_tx_never_used =
SameWorkerSender(same_worker_tx_never_used, Arc::new(AtomicU16::new(0)));
let (send_result_never_used, _send_result_rx_never_used) = mpsc::channel::<SendResult>(1);
let (send_result_never_used, _send_result_rx_never_used) = JobCompletedSender::new_never_used();
let label = if job.permissioned_as != format!("u/{}", job.created_by)
&& job.permissioned_as != job.created_by
@@ -2006,8 +2069,8 @@ async fn cancel_zombie_flow_job(
Ok(())
}
pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::Result<()> {
let hub_base_url = load_value_from_global_settings(db, HUB_BASE_URL_SETTING).await?;
pub async fn reload_hub_base_url_setting(conn: &Connection, server_mode: bool) -> error::Result<()> {
let hub_base_url = load_value_from_global_settings_with_conn(conn, HUB_BASE_URL_SETTING, true).await?;
let base_url = if let Some(q) = hub_base_url {
if let Ok(v) = serde_json::from_value::<String>(q.clone()) {
@@ -2030,16 +2093,18 @@ pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::R
let mut l = HUB_BASE_URL.write().await;
if server_mode {
#[cfg(feature = "embedding")]
if *l != base_url {
let disable_embedding = std::env::var("DISABLE_EMBEDDING")
.ok()
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false);
if !disable_embedding {
let db_clone = db.clone();
tokio::spawn(async move {
update_embeddings_db(&db_clone).await;
});
if let Some(db) = conn.as_sql() {
if *l != base_url {
let disable_embedding = std::env::var("DISABLE_EMBEDDING")
.ok()
.map(|x| x.parse::<bool>().unwrap_or(false))
.unwrap_or(false);
if !disable_embedding {
let db_clone = db.clone();
tokio::spawn(async move {
update_embeddings_db(&db_clone).await;
});
}
}
}
}
@@ -2048,9 +2113,9 @@ pub async fn reload_hub_base_url_setting(db: &DB, server_mode: bool) -> error::R
Ok(())
}
pub async fn reload_critical_error_channels_setting(db: &DB) -> error::Result<()> {
pub async fn reload_critical_error_channels_setting(conn: &DB) -> error::Result<()> {
let critical_error_channels =
load_value_from_global_settings(db, CRITICAL_ERROR_CHANNELS_SETTING).await?;
load_value_from_global_settings(conn, CRITICAL_ERROR_CHANNELS_SETTING).await?;
let critical_error_channels = if let Some(q) = critical_error_channels {
if let Ok(v) = serde_json::from_value::<Vec<CriticalErrorChannel>>(q.clone()) {
backend/tests/worker.rs
+7 -9
View File
@@ -137,7 +137,6 @@ impl ApiServer {
rx,
port_tx,
false,
#[cfg(feature = "smtp")]
format!("http://localhost:{}", addr.port()),
));
@@ -315,7 +314,7 @@ mod suspend_resume {
let second = completed.next().await.unwrap();
// print_job(second, &db).await;
let token = windmill_worker::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap();
let token = windmill_common::auth::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap();
let secret = reqwest::get(format!(
"http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}&approver=ruben"
))
@@ -418,7 +417,7 @@ mod suspend_resume {
/* ... and send a request resume it. */
let second = completed.next().await.unwrap();
let token = windmill_worker::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap();
let token = windmill_common::auth::create_token_for_owner(&db, "test-workspace", "u/test-user", "", 100, "", &Uuid::nil(), None).await.unwrap();
let secret = reqwest::get(format!(
"http://localhost:{port}/api/w/test-workspace/jobs/job_signature/{second}/0?token={token}"
))
@@ -1033,11 +1032,11 @@ fn spawn_test_worker(
priority: 0,
tags: (*wc).worker_tags.clone(),
}];
windmill_common::worker::make_suspended_pull_query(&wc).await;
windmill_common::worker::make_pull_query(&wc).await;
windmill_common::worker::store_suspended_pull_query(&wc).await;
windmill_common::worker::store_pull_query(&wc).await;
}
windmill_worker::run_worker(
&db,
&db.into(),
worker_instance,
worker_name,
1,
@@ -1046,7 +1045,6 @@ fn spawn_test_worker(
rx,
tx2,
&base_internal_url,
false,
)
.await
};
@@ -3843,7 +3841,7 @@ async fn test_result_format(db: Pool<Postgres>) {
let port = server.addr.port();
let token = windmill_worker::create_token_for_owner(
let token = windmill_common::auth::create_token_for_owner(
&db,
"test-workspace",
"u/test-user",
@@ -3884,7 +3882,7 @@ async fn test_result_format(db: Pool<Postgres>) {
assert_eq!(job_result.get(), correct_result);
let response = windmill_api::jobs::run_wait_result(
&db,
&db.into(),
Uuid::parse_str(ordered_result_job_id).unwrap(),
"test-workspace".to_string(),
None,
backend/windmill-api/Cargo.toml
+6 -3
View File
@@ -10,13 +10,14 @@ path = "src/lib.rs"
[features]
default = []
enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise"]
enterprise = ["windmill-queue/enterprise", "windmill-audit/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "windmill-worker/enterprise"]
stripe = []
agent_worker_server = []
enterprise_saml = ["dep:samael"]
benchmark = []
embedding = ["dep:tinyvector", "dep:hf-hub", "dep:tokenizers", "dep:candle-core", "dep:candle-transformers", "dep:candle-nn", "dep:half"]
parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet"]
prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus"]
parquet = ["dep:datafusion", "dep:object_store", "dep:url", "windmill-common/parquet", "windmill-worker/parquet"]
prometheus = ["windmill-common/prometheus", "windmill-queue/prometheus", "dep:prometheus", "windmill-worker/prometheus"]
openidconnect = ["dep:openidconnect"]
tantivy = ["dep:windmill-indexer"]
kafka = ["dep:rdkafka"]
@@ -40,8 +41,10 @@ windmill-audit.workspace = true
windmill-parser.workspace = true
windmill-parser-ts.workspace = true
windmill-parser-py.workspace = true
windmill-parser-py-imports.workspace = true
windmill-git-sync.workspace = true
windmill-indexer = { workspace = true, optional = true }
windmill-worker.workspace = true
tokio.workspace = true
anyhow.workspace = true
argon2.workspace = true
backend/windmill-api/openapi.yaml
+35
View File
@@ -10498,6 +10498,41 @@ paths:
items:
$ref: "#/components/schemas/AutoscalingEvent"
/agent_workers/create_agent_token:
post:
summary: create agent token
operationId: createAgentToken
tags:
- agent_workers
requestBody:
description: agent token
required: true
content:
application/json:
schema:
type: object
properties:
worker_group:
type: string
tags:
type: array
items:
type: string
exp:
type: integer
required:
- worker_group
- tags
- exp
responses:
"200":
description: agent token created
content:
application/json:
schema:
type: string
/w/{workspace}/acls/get/{kind}/{path}:
get:
summary: get granular acls
backend/windmill-api/src/agent_workers_ee.rs
+52
View File
@@ -0,0 +1,52 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2042
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::db::DB;
use axum::Router;
use serde::{Deserialize, Serialize};
pub fn global_service() -> Router {
Router::new()
}
pub fn workspaced_service(
db: DB,
_base_internal_url: String,
) -> (
Router,
Option<tokio::task::JoinHandle<()>>,
windmill_worker::JobCompletedSender,
) {
use windmill_common::worker::Connection;
use windmill_worker::JobCompletedSender;
let (job_completed_tx, _job_completed_rx) =
JobCompletedSender::new(&Connection::Sql(db.clone()), 100);
let router = Router::new();
(router, None, job_completed_tx)
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct AgentAuth {
pub worker_group: String,
pub suffix: Option<String>,
pub tags: Vec<String>,
pub exp: Option<usize>,
}
pub struct AgentCache {}
impl AgentCache {
pub fn new() -> Self {
AgentCache {}
}
}
backend/windmill-api/src/drafts.rs
+1 -1
View File
@@ -1,6 +1,6 @@
/*
* Author: Ruben Fiszel
* Copyright: Windmill Labs, Inc 2042
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
backend/windmill-api/src/jobs.rs
+13 -9
View File
@@ -26,7 +26,7 @@ use tower::ServiceBuilder;
use windmill_common::error::JsonResult;
use windmill_common::flow_status::{JobResult, RestartedFrom};
use windmill_common::jobs::{format_completed_job_result, format_result, ENTRYPOINT_OVERRIDE};
use windmill_common::worker::{CLOUD_HOSTED, TMP_DIR};
use windmill_common::worker::{Connection, CLOUD_HOSTED, TMP_DIR};
use windmill_common::scripts::PREVIEW_IS_CODEBASE_HASH;
use windmill_common::variables::get_workspace_key;
@@ -372,7 +372,6 @@ async fn cancel_job_api(
email: "anonymous".to_string(),
},
};
let (mut tx, job_option) = tokio::time::timeout(
std::time::Duration::from_secs(120),
windmill_queue::cancel_job(
@@ -587,7 +586,9 @@ async fn get_flow_job_debug_info(
Extension(db): Extension<DB>,
Path((w_id, id)): Path<(String, Uuid)>,
) -> error::Result<Response> {
let job = GetQuery::new().fetch_queued(&db, id, &w_id).await?;
let job = GetQuery::new()
.fetch_queued((&db).into(), id, &w_id)
.await?;
if let Some(job) = job {
let is_flow = job.is_flow();
if job.is_flow_step || !is_flow {
@@ -926,8 +927,9 @@ impl<'a> GetQuery<'a> {
// Try to fetch the code from the cache, fallback to the preview code.
// NOTE: This could check for the job kinds instead of the `or_else` but it's not
// necessary as `fetch_script` return early if the job kind is not a preview one.
cache::job::fetch_script(db, kind, hash)
.or_else(|_| cache::job::fetch_preview_script(db, &id, raw_lock, raw_code))
let conn = Connection::from(db.clone());
cache::job::fetch_script(db.clone(), kind, hash)
.or_else(|_| cache::job::fetch_preview_script(&conn, &id, raw_lock, raw_code))
.await
.ok()
.inspect(|data| {
@@ -956,7 +958,7 @@ impl<'a> GetQuery<'a> {
self.check_auth(job.as_ref().map(|job| job.created_by.as_str()))?;
if let Some(job) = job.as_mut() {
self.resolve_raw_values(db, job.id, job.job_kind, job.script_hash, job)
self.resolve_raw_values(&db, job.id, job.job_kind, job.script_hash, job)
.await;
}
if self.with_flow {
@@ -993,12 +995,14 @@ impl<'a> GetQuery<'a> {
self.resolve_raw_values(db, job.id, job.job_kind, job.script_hash, job)
.await;
}
if self.with_flow {
cjob = resolve_maybe_value(db, workspace_id, self.with_code, cjob, |job| {
job.raw_flow.as_mut()
})
.await?;
}
if let Some(mut cjob) = cjob {
cjob.inner = format_completed_job_result(cjob.inner);
return Ok(Some(cjob));
@@ -1008,7 +1012,7 @@ impl<'a> GetQuery<'a> {
async fn fetch(self, db: &DB, job_id: Uuid, workspace_id: &str) -> error::Result<Job> {
let cjob = self
.fetch_completed(db, job_id, workspace_id)
.fetch_completed(db.into(), job_id, workspace_id)
.await?
.map(Job::CompletedJob);
@@ -1016,7 +1020,7 @@ impl<'a> GetQuery<'a> {
Some(cjob) => Ok(cjob),
None => {
let job_maybe = self
.fetch_queued(db, job_id, workspace_id)
.fetch_queued(db.into(), job_id, workspace_id)
.await?
.map(Job::QueuedJob);
// potential race condition here, if the job was in queue and completed right after the fetch completed, so we need to check one last time
@@ -1024,7 +1028,7 @@ impl<'a> GetQuery<'a> {
return Ok(job);
} else {
let cjob2 = self
.fetch_completed(db, job_id, workspace_id)
.fetch_completed(db.into(), job_id, workspace_id)
.await?
.map(Job::CompletedJob);
not_found_if_none(cjob2, "Job", job_id.to_string())
backend/windmill-api/src/lib.rs
+58 -13
View File
@@ -25,6 +25,8 @@ use crate::{
webhook_util::WebhookShared,
};
#[cfg(feature = "agent_worker_server")]
use agent_workers_ee::AgentCache;
use anyhow::Context;
use argon2::Argon2;
use axum::extract::DefaultBodyLimit;
@@ -56,6 +58,8 @@ use windmill_common::{utils::GIT_VERSION, BASE_URL, INSTANCE_NAME};
use crate::scim_ee::has_scim_token;
use windmill_common::error::AppError;
#[cfg(feature = "agent_worker_server")]
mod agent_workers_ee;
mod ai;
mod apps;
pub mod args;
@@ -74,9 +78,9 @@ mod folders;
mod granular_acls;
mod groups;
#[cfg(feature = "http_trigger")]
mod http_triggers;
#[cfg(feature = "http_trigger")]
mod http_trigger_auth;
#[cfg(feature = "http_trigger")]
mod http_triggers;
mod indexer_ee;
mod inputs;
mod integration;
@@ -209,10 +213,10 @@ pub async fn run_server(
job_index_reader: Option<IndexReader>,
log_index_reader: Option<ServiceLogIndexReader>,
addr: SocketAddr,
mut rx: tokio::sync::broadcast::Receiver<()>,
mut killpill_rx: tokio::sync::broadcast::Receiver<()>,
port_tx: tokio::sync::oneshot::Sender<String>,
server_mode: bool,
#[cfg(feature = "smtp")] base_internal_url: String,
_base_internal_url: String,
) -> anyhow::Result<()> {
let user_db = UserDB::new(db.clone());
@@ -246,7 +250,10 @@ pub async fn run_server(
.layer(Extension(log_index_reader))
// .layer(Extension(index_writer))
.layer(CookieManagerLayer::new())
.layer(Extension(WebhookShared::new(rx.resubscribe(), db.clone())))
.layer(Extension(WebhookShared::new(
killpill_rx.resubscribe(),
db.clone(),
)))
.layer(DefaultBodyLimit::max(
REQUEST_SIZE_LIMIT.read().await.clone(),
));
@@ -279,7 +286,7 @@ pub async fn run_server(
db: db.clone(),
user_db: user_db,
auth_cache: auth_cache.clone(),
base_internal_url: base_internal_url.clone(),
base_internal_url: _base_internal_url.clone(),
});
if let Err(err) = smtp_server.start_listener_thread(addr).await {
tracing::error!("Error starting SMTP server: {err:#}");
@@ -385,41 +392,48 @@ pub async fn run_server(
if !*CLOUD_HOSTED && server_mode {
#[cfg(feature = "websocket")]
{
let ws_killpill_rx = rx.resubscribe();
let ws_killpill_rx = killpill_rx.resubscribe();
websocket_triggers::start_websockets(db.clone(), ws_killpill_rx);
}
#[cfg(all(feature = "enterprise", feature = "kafka"))]
{
let kafka_killpill_rx = rx.resubscribe();
let kafka_killpill_rx = killpill_rx.resubscribe();
kafka_triggers_ee::start_kafka_consumers(db.clone(), kafka_killpill_rx);
}
#[cfg(all(feature = "enterprise", feature = "nats"))]
{
let nats_killpill_rx = rx.resubscribe();
let nats_killpill_rx = killpill_rx.resubscribe();
nats_triggers_ee::start_nats_consumers(db.clone(), nats_killpill_rx);
}
#[cfg(feature = "postgres_trigger")]
{
let db_killpill_rx = rx.resubscribe();
let db_killpill_rx = killpill_rx.resubscribe();
postgres_triggers::start_database(db.clone(), db_killpill_rx);
}
#[cfg(feature = "mqtt_trigger")]
{
let mqtt_killpill_rx = rx.resubscribe();
let mqtt_killpill_rx = killpill_rx.resubscribe();
mqtt_triggers::start_mqtt_consumer(db.clone(), mqtt_killpill_rx);
}
#[cfg(all(feature = "enterprise", feature = "sqs_trigger"))]
{
let sqs_killpill_rx = rx.resubscribe();
let sqs_killpill_rx = killpill_rx.resubscribe();
sqs_triggers_ee::start_sqs(db.clone(), sqs_killpill_rx);
}
}
#[cfg(feature = "agent_worker_server")]
let (agent_workers_router, agent_workers_bg_processor, agent_workers_killpill_tx) =
agent_workers_ee::workspaced_service(db.clone(), _base_internal_url.clone());
#[cfg(feature = "agent_worker_server")]
let agent_cache = Arc::new(AgentCache::new());
// build our application with a route
let app = Router::new()
.nest(
@@ -495,6 +509,26 @@ pub async fn run_server(
.nest("/ai", ai::global_service())
.route_layer(from_extractor::<ApiAuthed>())
.route_layer(from_extractor::<users::Tokened>())
.nest("/agent_workers", {
#[cfg(feature = "agent_worker_server")]
{
agent_workers_ee::global_service().layer(Extension(agent_cache.clone()))
}
#[cfg(not(feature = "agent_worker_server"))]
{
Router::new()
}
})
.nest("/w/:workspace_id/agent_workers", {
#[cfg(feature = "agent_worker_server")]
{
agent_workers_router.layer(Extension(agent_cache.clone()))
}
#[cfg(not(feature = "agent_worker_server"))]
{
Router::new()
}
})
.nest("/jobs", jobs::global_root_service())
.nest(
"/srch/w/:workspace_id/index",
@@ -647,11 +681,22 @@ pub async fn run_server(
.expect("Failed to send port");
let server = server.with_graceful_shutdown(async move {
rx.recv().await.ok();
killpill_rx.recv().await.ok();
#[cfg(feature = "agent_worker_server")]
if let Err(e) = agent_workers_killpill_tx.kill().await {
tracing::error!("Error killing agent workers: {e:#}");
}
tracing::info!("Graceful shutdown of server");
});
server.await?;
#[cfg(feature = "agent_worker_server")]
if let Some(bg_processor) = agent_workers_bg_processor {
tracing::info!("server off. shutting down agent workers bg processor");
bg_processor.await?;
tracing::info!("agent workers bg processor shut down");
}
Ok(())
}
backend/windmill-api/src/resources.rs
+1 -1
View File
@@ -570,7 +570,7 @@ pub async fn transform_json_value<'c>(
};
let variables = variables::get_reserved_variables(
db,
&db.into(),
workspace,
token,
&job.email,
backend/windmill-api/src/variables.rs
+1 -1
View File
@@ -61,7 +61,7 @@ async fn list_contextual_variables(
) -> JsonResult<Vec<ContextualVariable>> {
Ok(Json(
get_reserved_variables(
&db,
&db.into(),
&w_id,
"q1A0qcPuO00yxioll7iph76N9CJDqn",
&email,
backend/windmill-common/Cargo.toml
+2
View File
@@ -34,6 +34,8 @@ serde_json.workspace = true
chrono.workspace = true
chrono-tz.workspace = true
hex.workspace = true
reqwest-middleware = { workspace = true }
reqwest-retry = { workspace = true }
rand.workspace = true
sqlx = { workspace = true, features = ["postgres"] }
uuid.workspace = true
backend/windmill-common/src/agent_workers.rs
+90
View File
@@ -0,0 +1,90 @@
use serde::{Deserialize, Serialize};
#[derive(Deserialize, Serialize)]
pub struct QueueInitJob {
pub content: String,
}
use lazy_static::lazy_static;
use std::time::Duration;
use reqwest_middleware::ClientBuilder;
use reqwest_retry::{policies::ExponentialBackoff, RetryTransientMiddleware};
use crate::{jwt::decode_without_verify, worker::HttpClient};
lazy_static! {
pub static ref BASE_INTERNAL_URL: String =
std::env::var("BASE_INTERNAL_URL").unwrap_or("http://localhost:8080".to_string());
pub static ref AGENT_TOKEN: String = std::env::var("AGENT_TOKEN").unwrap_or_default();
pub static ref DECODED_AGENT_TOKEN: Option<AgentAuth> = {
if AGENT_TOKEN.is_empty() {
None
} else {
decode_without_verify::<AgentAuth>(AGENT_TOKEN.trim_start_matches(AGENT_JWT_PREFIX))
.ok()
}
};
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct AgentAuth {
pub worker_group: String,
pub suffix: Option<String>,
pub tags: Vec<String>,
pub exp: Option<usize>,
}
pub const AGENT_JWT_PREFIX: &str = "jwt_agent_";
pub fn build_agent_http_client(worker_suffix: &str) -> HttpClient {
let client = ClientBuilder::new(
reqwest::Client::builder()
.pool_max_idle_per_host(10)
.pool_idle_timeout(Duration::from_secs(60))
.connect_timeout(Duration::from_secs(10))
.timeout(Duration::from_secs(30))
.default_headers({
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
"User-Agent", // Replace with your desired header name
"Windmill-Agent/1.0".parse().unwrap(), // Replace with your desired header value
);
let token = format!(
"{}{}_{}",
AGENT_JWT_PREFIX,
worker_suffix,
AGENT_TOKEN.trim_start_matches(AGENT_JWT_PREFIX),
);
headers.insert(
"Authorization",
format!("Bearer {}", token).parse().unwrap(),
);
headers
})
.build()
.expect("Failed to create HTTP client"),
)
.with(RetryTransientMiddleware::new_with_policy(
ExponentialBackoff::builder().build_with_max_retries(5),
))
.build();
HttpClient(client)
}
#[derive(Deserialize, Serialize)]
pub struct PingJobStatus {
pub mem_peak: Option<i32>,
pub current_mem: Option<i32>,
}
#[derive(Deserialize, Serialize, Debug)]
pub struct PingJobStatusResponse {
pub canceled_by: Option<String>,
pub canceled_reason: Option<String>,
pub already_completed: bool,
}
// #[derive(Serialize, Deserialize)]
// pub struct PullJobRequest {
// pub worker_name: String,
// }
backend/windmill-common/src/auth.rs
+63 -1
View File
@@ -1,8 +1,11 @@
use anyhow::Context;
use serde::{Deserialize, Serialize};
use uuid::Uuid;
use crate::{
db::Authed,
error::{Error, Result},
jwt,
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL, SUPERADMIN_SYNC_EMAIL},
DB,
};
@@ -22,7 +25,7 @@ pub struct JWTAuthClaims {
pub scopes: Option<Vec<String>>,
}
#[derive(Deserialize)]
#[derive(Deserialize, Debug)]
pub struct JobPerms {
pub email: String,
pub username: String,
@@ -205,3 +208,62 @@ pub async fn get_groups_for_user(
.collect();
Ok(groups)
}
#[tracing::instrument(level = "trace", skip_all)]
pub async fn create_token_for_owner(
db: &DB,
w_id: &str,
owner: &str,
label: &str,
expires_in: u64,
email: &str,
job_id: &Uuid,
perms: Option<JobPerms>,
) -> crate::error::Result<String> {
let job_perms = if perms.is_some() {
Ok(perms)
} else {
sqlx::query_as!(
JobPerms,
"SELECT email, username, is_admin, is_operator, groups, folders FROM job_perms WHERE job_id = $1 AND workspace_id = $2",
job_id,
w_id
)
.fetch_optional(db)
.await
};
let job_authed = match job_perms {
Ok(Some(jp)) => jp.into(),
_ => {
tracing::warn!("Could not get permissions for job {job_id} from job_perms table, getting permissions directly...");
fetch_authed_from_permissioned_as(owner.to_string(), email.to_string(), w_id, db)
.await
.map_err(|e| {
Error::internal_err(format!(
"Could not get permissions directly for job {job_id}: {e:#}"
))
})?
}
};
let payload = JWTAuthClaims {
email: job_authed.email,
username: job_authed.username,
is_admin: job_authed.is_admin,
is_operator: job_authed.is_operator,
groups: job_authed.groups,
folders: job_authed.folders,
label: Some(label.to_string()),
workspace_id: w_id.to_string(),
exp: (chrono::Utc::now() + chrono::Duration::seconds(expires_in as i64)).timestamp()
as usize,
job_id: Some(job_id.to_string()),
scopes: None,
};
let token = jwt::encode_with_internal_secret(&payload)
.await
.with_context(|| format!("Could not encode JWT token for job {job_id}"))?;
Ok(format!("jwt_{}", token))
}
backend/windmill-common/src/cache.rs
+182 -116
View File
@@ -7,7 +7,6 @@
//! This shall only be used for testing, e.g. [`sqlx::test`] spawn a database per test,
//! and there is only one test per thread, so using thread-local cache avoid unexpected results.
use anyhow::anyhow;
use crate::{
apps::AppScriptId,
error,
@@ -15,6 +14,7 @@ use crate::{
schema::SchemaValidator,
scripts::{ScriptHash, ScriptLang},
};
use anyhow::anyhow;
#[cfg(feature = "scoped_cache")]
use std::thread::ThreadId;
@@ -322,6 +322,25 @@ pub struct RawScript {
pub meta: Option<ScriptMetadata>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct RawScriptApi {
pub content: String,
pub lock: Option<String>,
pub meta: Option<ScriptMetadata>,
}
impl From<RawScript> for RawScriptApi {
fn from(value: RawScript) -> Self {
RawScriptApi { content: value.content, lock: value.lock, meta: value.meta }
}
}
impl From<RawScriptApi> for RawScript {
fn from(value: RawScriptApi) -> Self {
RawScript { content: value.content, lock: value.lock, meta: value.meta }
}
}
#[derive(Debug)]
pub struct RawFlow {
pub raw_flow: Box<RawValue>,
@@ -334,6 +353,25 @@ pub struct RawNode {
pub raw_flow: Option<Box<RawValue>>,
}
#[derive(Debug, Deserialize, Serialize)]
pub struct RawNodeApi {
pub raw_code: Option<String>,
pub raw_lock: Option<String>,
pub raw_flow: Option<Box<RawValue>>,
}
impl From<RawNode> for RawNodeApi {
fn from(value: RawNode) -> Self {
RawNodeApi { raw_code: value.raw_code, raw_lock: value.raw_lock, raw_flow: value.raw_flow }
}
}
impl From<RawNodeApi> for RawNode {
fn from(value: RawNodeApi) -> Self {
RawNode { raw_code: value.raw_code, raw_lock: value.raw_lock, raw_flow: value.raw_flow }
}
}
#[derive(Debug, Clone)]
struct Entry<T>(Arc<T>);
@@ -343,7 +381,7 @@ struct ScriptFull {
pub meta: Arc<ScriptMetadata>,
}
fn unwrap_or_error<Key: std::fmt::Debug, Val>(
pub fn unwrap_or_error<Key: std::fmt::Debug, Val>(
at: &'static Location,
entity: &'static str,
key: Key,
@@ -363,6 +401,11 @@ pub fn clear() {
}
pub mod flow {
use crate::{
worker::{fetch_flow_node_query, Connection},
DB,
};
use super::*;
make_static! {
@@ -388,10 +431,10 @@ pub mod flow {
/// This should be preferred over fetching the database directly.
#[track_caller]
pub fn fetch_script<'c>(
e: impl PgExecutor<'c>,
conn: &'c Connection,
node: FlowNodeId,
) -> impl Future<Output = error::Result<Arc<ScriptData>>> {
let fetch_node = fetch_node(e, node);
) -> impl Future<Output = error::Result<Arc<ScriptData>>> + 'c {
let fetch_node = fetch_node(conn, node);
async move {
fetch_node.await.and_then(|data| match data {
RawData::Script(data) => Ok(data),
@@ -409,11 +452,12 @@ pub mod flow {
/// This should be preferred over fetching the database directly.
#[track_caller]
pub fn fetch_flow<'c>(
e: impl PgExecutor<'c>,
db: &'c DB,
node: FlowNodeId,
) -> impl Future<Output = error::Result<Arc<FlowData>>> {
let fetch_node = fetch_node(e, node);
) -> impl Future<Output = error::Result<Arc<FlowData>>> + 'c {
async move {
let conn = Connection::Sql(db.clone());
let fetch_node = fetch_node(&conn, node);
fetch_node.await.and_then(|data| match data {
RawData::Flow(data) => Ok(data),
RawData::Script(_) => Err(error::Error::internal_err(format!(
@@ -430,31 +474,23 @@ pub mod flow {
/// This should be preferred over fetching the database directly.
#[track_caller]
pub(super) fn fetch_node<'c>(
e: impl PgExecutor<'c>,
conn: &'c Connection,
node: FlowNodeId,
) -> impl Future<Output = error::Result<RawData>> {
) -> impl Future<Output = error::Result<RawData>> + 'c {
let loc = Location::caller();
// If not present, `get_or_insert_async` will lock the key until the future completes,
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
NODES.get_or_insert_async(node, async move {
sqlx::query!(
"SELECT \
code AS \"raw_code: String\", \
lock AS \"raw_lock: String\", \
flow AS \"raw_flow: Json<Box<RawValue>>\" \
FROM flow_node WHERE id = $1 LIMIT 1",
node.0,
)
.fetch_optional(e)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Flow node", node))
.map(|r| RawNode {
raw_code: r.raw_code,
raw_lock: r.raw_lock,
raw_flow: r.raw_flow.map(|Json(raw_flow)| raw_flow),
})
match conn {
Connection::Sql(db) => fetch_flow_node_query(db, node.0, loc).await,
Connection::Http(client) => {
let r = client
.get::<RawNodeApi>(&format!("/api/agent_workers/flow_script/{}", node.0))
.await?;
Ok(r.into())
}
}
})
}
@@ -502,6 +538,8 @@ pub mod flow {
}
pub mod script {
use crate::{worker::Connection, DB};
use super::*;
make_static! {
@@ -520,66 +558,80 @@ pub mod script {
/// it to the file system and cache.
/// This should be preferred over fetching the database directly.
#[track_caller]
pub fn fetch<'c>(
e: impl PgExecutor<'c>,
pub fn fetch(
conn: &Connection,
hash: ScriptHash,
) -> impl Future<Output = error::Result<(Arc<ScriptData>, Arc<ScriptMetadata>)>> {
// If not present, `get_or_insert_async` will lock the key until the future completes,
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
let loc = Location::caller();
let conn = conn.clone();
let fut = CACHE.get_or_insert_async(hash, async move {
sqlx::query!(
"SELECT \
content AS \"content!: String\",
lock AS \"lock: String\", \
language AS \"language: Option<ScriptLang>\", \
envs AS \"envs: Vec<String>\", \
schema AS \"schema: String\", \
schema_validation AS \"schema_validation: bool\", \
codebase LIKE '%.tar' as use_tar \
FROM script WHERE hash = $1 LIMIT 1",
hash.0
)
.fetch_optional(e)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Script", hash))
.and_then(|r| {
Ok(RawScript {
content: r.content,
lock: r.lock,
meta: Some(ScriptMetadata {
language: r.language,
envs: r.envs,
codebase: if let Some(use_tar) = r.use_tar {
let sh = hash.to_string();
if use_tar {
Some(format!("{sh}.tar"))
} else {
Some(sh)
}
} else {
None
},
schema_validator: if r.schema_validation {
r.schema
.as_ref()
.map(|schema_str| {
SchemaValidator::from_schema(schema_str).map_err(|e| anyhow!("Couldn't create schema validator for script requiring schema validation: {e}"))
})
.transpose()?
} else {
None
},
schema: r.schema,
}),
})
})
match conn {
Connection::Sql(db) => fetch_script_from_db(&db, hash, loc).await,
Connection::Http(_) => Err(error::Error::InternalErr(format!(
"Cannot fetch script in HTTP mode"
))),
}
});
fut.map_ok(|ScriptFull { data, meta }| (data, meta))
}
async fn fetch_script_from_db(
db: &DB,
hash: ScriptHash,
loc: &'static Location<'_>,
) -> error::Result<RawScript> {
sqlx::query!(
"SELECT \
content AS \"content!: String\",
lock AS \"lock: String\", \
language AS \"language: Option<ScriptLang>\", \
envs AS \"envs: Vec<String>\", \
schema AS \"schema: String\", \
schema_validation AS \"schema_validation: bool\", \
codebase LIKE '%.tar' as use_tar \
FROM script WHERE hash = $1 LIMIT 1",
hash.0
)
.fetch_optional(db)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Script", hash))
.and_then(|r| {
Ok(RawScript {
content: r.content,
lock: r.lock,
meta: Some(ScriptMetadata {
language: r.language,
envs: r.envs,
codebase: if let Some(use_tar) = r.use_tar {
let sh = hash.to_string();
if use_tar {
Some(format!("{sh}.tar"))
} else {
Some(sh)
}
} else {
None
},
schema_validator: if r.schema_validation {
r.schema
.as_ref()
.map(|schema_str| {
SchemaValidator::from_schema(schema_str).map_err(|e| anyhow!("Couldn't create schema validator for script requiring schema validation: {e}"))
})
.transpose()?
} else {
None
},
schema: r.schema,
}),
})
})
}
/// Invalidate the script cache for the given `hash`.
pub fn invalidate(hash: ScriptHash) {
let _ = CACHE.remove(&hash);
@@ -587,6 +639,8 @@ pub mod script {
}
pub mod app {
use crate::worker::{fetch_raw_script_from_app_query, Connection};
use super::*;
make_static! {
@@ -605,23 +659,23 @@ pub mod app {
/// This should be preferred over fetching the database directly.
#[track_caller]
pub fn fetch_script<'c>(
e: impl PgExecutor<'c>,
conn: &'c Connection,
id: AppScriptId,
) -> impl Future<Output = error::Result<Arc<ScriptData>>> {
) -> impl Future<Output = error::Result<Arc<ScriptData>>> + 'c {
// If not present, `get_or_insert_async` will lock the key until the future completes,
// so only one thread will be able to fetch the data from the database and write it to
// the file system and cache, hence no race on the file system.
let loc = Location::caller();
let fut = CACHE.get_or_insert_async(id, async move {
sqlx::query!(
"SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1",
id.0,
)
.fetch_optional(e)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Application script", id))
.map(|r| RawScript { content: r.code, lock: r.lock, meta: None })
match conn {
Connection::Sql(db) => fetch_raw_script_from_app_query(db, id.0, loc).await,
Connection::Http(client) => {
let r = client
.get::<RawScriptApi>(&format!("/api/agent_workers/app_script/{}", id.0))
.await?;
Ok(r.into())
}
}
});
fut.map_ok(|Entry(data)| data)
}
@@ -629,7 +683,7 @@ pub mod app {
pub mod job {
use super::*;
use crate::jobs::JobKind;
use crate::{jobs::JobKind, worker::Connection, DB};
#[cfg(not(feature = "scoped_cache"))]
lazy_static! {
@@ -649,15 +703,18 @@ pub mod job {
}
#[track_caller]
pub fn fetch_preview_flow<'a, 'c>(
e: impl PgExecutor<'c> + 'a,
pub fn fetch_preview_flow<'a>(
db: &'a DB,
job: &'a Uuid,
// original raw values from `queue` or `completed_job` tables:
// kept for backward compatibility.
raw_flow: Option<Json<Box<RawValue>>>,
) -> impl Future<Output = error::Result<Arc<FlowData>>> + 'a {
let fetch_preview = fetch_preview(e, job, None, None, raw_flow);
// Create the Connection first so it lives for the entire scope
async move {
let conn = Connection::from(db);
let fetch_preview = fetch_preview(&conn, job, None, None, raw_flow);
fetch_preview.await.and_then(|data| match data {
RawData::Flow(data) => Ok(data),
RawData::Script(_) => Err(error::Error::internal_err(format!(
@@ -669,7 +726,7 @@ pub mod job {
#[track_caller]
pub fn fetch_preview_script<'a, 'c>(
e: impl PgExecutor<'c> + 'a,
e: &'a Connection,
job: &'a Uuid,
// original raw values from `queue` or `completed_job` tables:
// kept for backward compatibility.
@@ -689,7 +746,7 @@ pub mod job {
#[track_caller]
pub fn fetch_preview<'a, 'c>(
e: impl PgExecutor<'c> + 'a,
e: &'a Connection,
job: &'a Uuid,
// original raw values from `queue` or `completed_job` tables:
// kept for backward compatibility.
@@ -700,16 +757,21 @@ pub mod job {
let loc = Location::caller();
let fetch = async move {
match (raw_lock, raw_code, raw_flow) {
(None, None, None) => sqlx::query!(
"SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json<Box<RawValue>>\" \
FROM v2_job WHERE id = $1 LIMIT 1",
job
)
.fetch_optional(e)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Preview", job))
.map(|r| (r.raw_lock, r.raw_code, r.raw_flow)),
(None, None, None) => match e {
Connection::Sql(pool) => sqlx::query!(
"SELECT raw_code, raw_lock, raw_flow AS \"raw_flow: Json<Box<RawValue>>\" \
FROM v2_job WHERE id = $1 LIMIT 1",
job
)
.fetch_optional(pool)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Preview", job))
.map(|r| (r.raw_lock, r.raw_code, r.raw_flow)),
Connection::Http(_) => Err(error::Error::InternalErr(format!(
"Cannot fetch preview in HTTP mode"
))),
},
(lock, code, flow) => Ok((lock, code, flow)),
}
.and_then(|(lock, code, flow)| match flow {
@@ -730,8 +792,8 @@ pub mod job {
}
#[track_caller]
pub fn fetch_script<'c>(
e: impl PgExecutor<'c>,
pub fn fetch_script(
db: DB,
kind: JobKind,
hash: Option<ScriptHash>,
) -> impl Future<Output = error::Result<Arc<ScriptData>>> {
@@ -739,11 +801,15 @@ pub mod job {
let loc = Location::caller();
async move {
match (kind, hash.map(|ScriptHash(id)| id)) {
(FlowScript, Some(id)) => flow::fetch_script(e, FlowNodeId(id)).await,
(Script | Dependencies, Some(hash)) => script::fetch(e, ScriptHash(hash))
(FlowScript, Some(id)) => {
flow::fetch_script(&Connection::Sql(db.clone()), FlowNodeId(id)).await
}
(Script | Dependencies, Some(hash)) => script::fetch(&db.into(), ScriptHash(hash))
.await
.map(|(data, _meta)| data),
(AppScript, Some(id)) => app::fetch_script(e, AppScriptId(id)).await,
(AppScript, Some(id)) => {
app::fetch_script(&Connection::Sql(db.clone()), AppScriptId(id)).await
}
_ => Err(error::Error::internal_err(format!(
"Isn't a script job: {:?}",
kind
@@ -755,19 +821,19 @@ pub mod job {
#[track_caller]
pub fn fetch_flow<'c>(
e: impl PgExecutor<'c> + Copy,
db: &'c DB,
kind: JobKind,
hash: Option<ScriptHash>,
) -> impl Future<Output = error::Result<Arc<FlowData>>> {
) -> impl Future<Output = error::Result<Arc<FlowData>>> + 'c {
use JobKind::*;
let loc = Location::caller();
async move {
match (kind, hash.map(|ScriptHash(id)| id)) {
(FlowDependencies, Some(id)) => flow::fetch_version(e, id).await,
(FlowNode, Some(id)) => flow::fetch_flow(e, FlowNodeId(id)).await,
(Flow, Some(id)) => match flow::fetch_version_lite(e, id).await {
(FlowDependencies, Some(id)) => flow::fetch_version(db, id).await,
(FlowNode, Some(id)) => flow::fetch_flow(db, FlowNodeId(id)).await,
(Flow, Some(id)) => match flow::fetch_version_lite(db, id).await {
Ok(raw_flow) => Ok(raw_flow),
Err(_) => flow::fetch_version(e, id).await,
Err(_) => flow::fetch_version(db, id).await,
},
_ => Err(error::Error::internal_err(format!(
"Isn't a flow job {:?}",
backend/windmill-common/src/flows.rs
+8 -8
View File
@@ -22,7 +22,7 @@ use crate::{
error::Error,
more_serde::{default_empty_string, default_id, default_null, default_true, is_default},
scripts::{Schema, ScriptHash, ScriptLang},
worker::to_raw_value,
worker::{to_raw_value, Connection}, DB,
};
#[derive(Serialize, Deserialize, sqlx::FromRow)]
@@ -731,7 +731,7 @@ pub async fn resolve_maybe_value<T>(
}
/// Resolve modules recursively.
pub async fn resolve_value(
async fn resolve_value(
e: &sqlx::PgPool,
workspace_id: &str,
value: &mut Box<JsonRawValue>,
@@ -749,7 +749,7 @@ pub async fn resolve_value(
/// Resolve module value recursively.
pub async fn resolve_module(
e: &sqlx::PgPool,
db: &DB,
workspace_id: &str,
value: &mut Box<JsonRawValue>,
with_code: bool,
@@ -783,7 +783,7 @@ pub async fn resolve_module(
let (lock, content) = if !with_code {
(Some("...".to_string()), "...".to_string())
} else {
cache::flow::fetch_script(e, id)
cache::flow::fetch_script(&Connection::Sql(db.clone()), id)
.await
.map(|data| (data.lock.clone(), data.code.clone()))?
};
@@ -801,13 +801,13 @@ pub async fn resolve_module(
};
}
ForloopFlow { modules, modules_node, .. } | WhileloopFlow { modules, modules_node, .. } => {
resolve_modules(e, workspace_id, modules, modules_node.take(), with_code).await?;
resolve_modules(db, workspace_id, modules, modules_node.take(), with_code).await?;
}
BranchOne { branches, default, default_node } => {
resolve_modules(e, workspace_id, default, default_node.take(), with_code).await?;
resolve_modules(db, workspace_id, default, default_node.take(), with_code).await?;
for branch in branches {
resolve_modules(
e,
db,
workspace_id,
&mut branch.modules,
branch.modules_node.take(),
@@ -819,7 +819,7 @@ pub async fn resolve_module(
BranchAll { branches, .. } => {
for branch in branches {
resolve_modules(
e,
db,
workspace_id,
&mut branch.modules,
branch.modules_node.take(),
backend/windmill-common/src/jobs.rs
+9 -1
View File
@@ -10,6 +10,7 @@ use tokio::io::AsyncReadExt;
use uuid::Uuid;
pub const ENTRYPOINT_OVERRIDE: &str = "_ENTRYPOINT_OVERRIDE";
pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000;
use crate::{
apps::AppScriptId,
@@ -24,7 +25,7 @@ use crate::{
#[derive(sqlx::Type, Serialize, Deserialize, Debug, PartialEq, Copy, Clone)]
#[sqlx(type_name = "JOB_KIND", rename_all = "lowercase")]
#[serde(rename_all(serialize = "lowercase"))]
#[serde(rename_all(serialize = "lowercase", deserialize = "lowercase"))]
pub enum JobKind {
Script,
#[allow(non_camel_case_types)]
@@ -51,6 +52,13 @@ impl JobKind {
JobKind::Flow | JobKind::FlowPreview | JobKind::SingleScriptFlow | JobKind::FlowNode
)
}
pub fn is_dependency(&self) -> bool {
matches!(
self,
JobKind::FlowDependencies | JobKind::AppDependencies | JobKind::Dependencies
)
}
}
#[derive(sqlx::FromRow, Debug, Serialize, Clone)]
backend/windmill-common/src/jwt.rs
+18 -1
View File
@@ -1,6 +1,6 @@
use crate::error::{self, to_anyhow, Error};
use serde::{de::DeserializeOwned, Serialize};
use std::sync::Arc;
use std::{collections::HashSet, sync::Arc};
use tokio::sync::RwLock;
lazy_static::lazy_static! {
@@ -40,3 +40,20 @@ pub async fn decode_with_internal_secret<T: DeserializeOwned>(token: &str) -> er
Ok(result.claims)
}
pub fn decode_without_verify<T: DeserializeOwned>(token: &str) -> anyhow::Result<T> {
// Create a validation that skips all checks
let mut validation = jsonwebtoken::Validation::default();
validation.insecure_disable_signature_validation();
validation.validate_exp = false;
validation.validate_nbf = false;
validation.required_spec_claims = HashSet::new();
// Use an empty key since we're not verifying
let key = jsonwebtoken::DecodingKey::from_secret(&[]);
// Decode the token
let token_data = jsonwebtoken::decode::<T>(token, &key, &validation)?;
Ok(token_data.claims)
}
backend/windmill-common/src/lib.rs
+2 -1
View File
@@ -22,6 +22,7 @@ use error::Error;
use scripts::ScriptLang;
use sqlx::{Pool, Postgres};
pub mod agent_workers;
pub mod apps;
pub mod auth;
pub mod cache;
@@ -45,6 +46,7 @@ pub mod otel_ee;
pub mod queue;
pub mod s3_helpers;
pub mod schedule;
pub mod schema;
pub mod scripts;
pub mod server;
pub mod stats_ee;
@@ -55,7 +57,6 @@ pub mod utils;
pub mod variables;
pub mod worker;
pub mod workspaces;
pub mod schema;
pub const DEFAULT_MAX_CONNECTIONS_SERVER: u32 = 50;
pub const DEFAULT_MAX_CONNECTIONS_WORKER: u32 = 5;
backend/windmill-common/src/utils.rs
+35 -3
View File
@@ -23,7 +23,7 @@ use rand::distr::Alphanumeric;
use rand::{rng, Rng};
use reqwest::Client;
use semver::Version;
use serde::{Deserialize, Serialize};
use serde::{Deserialize, Deserializer, Serialize};
use sha2::{Digest, Sha256};
use sqlx::{Pool, Postgres};
use std::str::FromStr;
@@ -75,8 +75,8 @@ lazy_static::lazy_static! {
if std::env::var("BASE_INTERNAL_URL").is_err() {
panic!("BASE_INTERNAL_URL is required in agent mode")
}
if std::env::var("JOB_TOKEN").is_err() {
println!("JOB_TOKEN is not passed, hence workers will still need to create permissions for each job and the DATABASE_URL needs to be of a role that can INSERT into the job_perms table")
if std::env::var("AGENT_TOKEN").is_err() {
println!("AGENT_TOKEN is not passed. This is required for the agent to work and contains the JWT to authenticate with the server.")
}
#[cfg(not(feature = "enterprise"))]
@@ -175,6 +175,28 @@ pub fn hostname() -> String {
})
}
fn instance_name(hostname: &str) -> String {
hostname
.replace(" ", "")
.split("-")
.last()
.unwrap()
.to_ascii_lowercase()
.to_string()
}
pub fn worker_suffix(hostname: &str, rd_string: &str) -> String {
format!("{}-{}", instance_name(hostname), rd_string)
}
pub fn worker_name_with_suffix(is_agent: bool, worker_group: &str, suffix: &str) -> String {
if is_agent {
format!("ag-{}-{}", worker_group, suffix)
} else {
format!("wk-{}-{}", worker_group, suffix)
}
}
pub fn paginate(pagination: Pagination) -> (usize, usize) {
let per_page = pagination
.per_page
@@ -396,6 +418,16 @@ pub async fn report_critical_error(
}
}
pub fn empty_string_as_none<'de, D>(
deserializer: D,
) -> std::result::Result<Option<String>, D::Error>
where
D: Deserializer<'de>,
{
let option = <Option<String> as serde::Deserialize>::deserialize(deserializer)?;
Ok(option.filter(|s| !s.is_empty()))
}
pub async fn report_recovered_critical_error(
message: String,
_db: DB,
backend/windmill-common/src/variables.rs
+17 -13
View File
@@ -7,6 +7,7 @@
*/
use crate::error;
use crate::worker::Connection;
use crate::{worker::WORKER_GROUP, BASE_URL, DB};
use chrono::{SecondsFormat, Utc};
use magic_crypt::{MagicCrypt256, MagicCryptError, MagicCryptTrait};
@@ -166,7 +167,7 @@ lazy_static::lazy_static! {
}
pub async fn get_reserved_variables(
db: &DB,
conn: &Connection,
w_id: &str,
token: &str,
email: &str,
@@ -206,7 +207,7 @@ pub async fn get_reserved_variables(
}
};
let custom_envs = get_cached_workspace_envs(db, w_id).await;
let custom_envs = get_cached_workspace_envs(conn, w_id).await;
let joined_schedule_path = schedule_path
.clone()
@@ -354,10 +355,7 @@ pub async fn get_reserved_variables(
).collect()
}
async fn get_cached_workspace_envs(
db: &sqlx::Pool<sqlx::Postgres>,
w_id: &str,
) -> Vec<(String, String)> {
async fn get_cached_workspace_envs(conn: &Connection, w_id: &str) -> Vec<(String, String)> {
let cached_envs_o = CUSTOM_ENVS_CACHE.get(w_id).and_then(|(ts, envs)| {
if ts > chrono::Utc::now().timestamp() - (60 * 15) {
Some(envs)
@@ -369,13 +367,19 @@ async fn get_cached_workspace_envs(
let custom_envs = if let Some(cached_envs) = cached_envs_o {
cached_envs
} else {
let custom_envs = sqlx::query_as::<_, (String, String)>(
"SELECT name, value FROM workspace_env WHERE workspace_id = $1",
)
.bind(w_id)
.fetch_all(db)
.await
.unwrap_or_default();
let custom_envs = match conn {
Connection::Sql(db) => sqlx::query_as::<_, (String, String)>(
"SELECT name, value FROM workspace_env WHERE workspace_id = $1",
)
.bind(w_id)
.fetch_all(db)
.await
.unwrap_or_default(),
Connection::Http(client) => client
.get(&format!("/api/w/{w_id}/agent_workers/custom_envs"))
.await
.unwrap_or_default(),
};
CUSTOM_ENVS_CACHE.insert(
w_id.to_string(),
(chrono::Utc::now().timestamp(), custom_envs.clone()),
backend/windmill-common/src/worker.rs
+484 -54
View File
@@ -3,28 +3,55 @@ use bytes::Bytes;
use const_format::concatcp;
use itertools::Itertools;
use regex::Regex;
use reqwest_middleware::ClientWithMiddleware;
use semver::Version;
use serde::{Deserialize, Serialize};
use serde::{de::DeserializeOwned, Deserialize, Serialize};
use serde_json::value::RawValue;
use sqlx::{types::Json, Pool, Postgres};
use std::{
cmp::Reverse,
collections::{HashMap, HashSet},
fs::{self, File},
io::Write,
panic::Location,
path::{Component, Path, PathBuf},
str::FromStr,
sync::{atomic::AtomicBool, Arc},
};
use tokio::sync::RwLock;
use uuid::Uuid;
use windmill_macros::annotations;
use crate::{
error, global_settings::CUSTOM_TAGS_SETTING, indexer::TantivyIndexerSettings, server::Smtp,
agent_workers::{PingJobStatusResponse, BASE_INTERNAL_URL},
cache::{unwrap_or_error, RawNode, RawScript},
error::{self, to_anyhow},
global_settings::CUSTOM_TAGS_SETTING,
indexer::TantivyIndexerSettings,
server::Smtp,
KillpillSender, DB,
};
pub const DEFAULT_CLOUD_TIMEOUT: u64 = 900;
pub const DEFAULT_SELFHOSTED_TIMEOUT: u64 = 604800; // 7 days
lazy_static::lazy_static! {
pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| "default".to_string());
pub static ref WORKER_GROUP: String = std::env::var("WORKER_GROUP").unwrap_or_else(|_| {
#[cfg(not(feature = "enterprise"))]
{
"default".to_string()
}
#[cfg(feature = "enterprise")]
{
if let Some(token) = crate::agent_workers::DECODED_AGENT_TOKEN.as_ref() {
token.worker_group.clone()
} else {
"default".to_string()
}
}
});
pub static ref NO_LOGS: bool = std::env::var("NO_LOGS").ok().is_some_and(|x| x == "1" || x == "true");
pub static ref CGROUP_V2_PATH_RE: Regex = Regex::new(r#"(?m)^0::(/.*)$"#).unwrap();
@@ -61,6 +88,15 @@ lazy_static::lazy_static! {
pub static ref DEFAULT_TAGS_PER_WORKSPACE: AtomicBool = AtomicBool::new(false);
pub static ref DEFAULT_TAGS_WORKSPACES: Arc<RwLock<Option<Vec<String>>>> = Arc::new(RwLock::new(None));
pub static ref MAX_TIMEOUT: u64 = std::env::var("TIMEOUT")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or_else(|| if *CLOUD_HOSTED { DEFAULT_CLOUD_TIMEOUT } else { DEFAULT_SELFHOSTED_TIMEOUT });
pub static ref SCRIPT_TOKEN_EXPIRY: u64 = std::env::var("SCRIPT_TOKEN_EXPIRY")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or(*MAX_TIMEOUT);
pub static ref WORKER_CONFIG: Arc<RwLock<WorkerConfig>> = Arc::new(RwLock::new(WorkerConfig {
worker_tags: Default::default(),
@@ -115,6 +151,90 @@ pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/");
pub static MIN_VERSION_IS_LATEST: AtomicBool = AtomicBool::new(false);
#[derive(Clone)]
pub struct HttpClient(pub ClientWithMiddleware);
impl HttpClient {
pub async fn post<T: Serialize, R: DeserializeOwned>(
&self,
url: &str,
body: &T,
) -> anyhow::Result<R> {
let response = self
.0
.post(format!("{}{}", *BASE_INTERNAL_URL, url))
.json(body)
.send()
.await
.map_err(|e| anyhow::anyhow!(e))?;
let status = response.status();
if status.is_success() {
Ok(response.json().await?)
} else {
Err(anyhow::anyhow!(format!(
"HTTP agent request POST {} failed {}",
url,
response.status()
)))
}
}
pub async fn get<R: DeserializeOwned>(&self, url: &str) -> anyhow::Result<R> {
let response = self
.0
.get(format!("{}{}", *BASE_INTERNAL_URL, url))
.send()
.await
.map_err(|e| anyhow::anyhow!(e))?;
let status = response.status();
if status.is_success() {
Ok(response.json().await?)
} else {
Err(anyhow::anyhow!(format!(
"HTTP agent request GET {} failed {}",
url,
response.status()
)))
}
}
}
#[derive(Clone)]
pub enum Connection {
Sql(Pool<Postgres>),
Http(HttpClient),
}
impl std::fmt::Debug for Connection {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Connection::Sql(_) => write!(f, "Sql"),
Connection::Http(_) => write!(f, "Http"),
}
}
}
impl Connection {
pub fn as_sql(&self) -> Option<&Pool<Postgres>> {
match self {
Connection::Sql(db) => Some(db),
Connection::Http(_) => None,
}
}
}
impl From<Pool<Postgres>> for Connection {
fn from(value: Pool<Postgres>) -> Self {
Connection::Sql(value)
}
}
impl From<&Pool<Postgres>> for Connection {
fn from(value: &Pool<Postgres>) -> Self {
Connection::Sql(value.clone())
}
}
fn format_pull_query(peek: String) -> String {
let r = format!(
"WITH peek AS (
@@ -164,40 +284,48 @@ fn format_pull_query(peek: String) -> String {
r
}
pub async fn make_suspended_pull_query(wc: &WorkerConfig) {
if wc.worker_tags.len() == 0 {
tracing::error!("Empty tags in worker tags, skipping");
return;
}
let query = format_pull_query(format!(
pub fn make_suspended_pull_query(tags: &[String]) -> String {
format_pull_query(format!(
"SELECT id
FROM v2_job_queue
WHERE suspend_until IS NOT NULL AND (suspend <= 0 OR suspend_until <= now()) AND tag IN ({})
ORDER BY priority DESC NULLS LAST, created_at
FOR UPDATE SKIP LOCKED
LIMIT 1",
wc.worker_tags.iter().map(|x| format!("'{x}'")).join(", ")
));
tags.iter().map(|x| format!("'{x}'")).join(", ")
))
}
// pub async fn make_suspended
pub async fn store_suspended_pull_query(wc: &WorkerConfig) {
if wc.worker_tags.len() == 0 {
tracing::error!("Empty tags in worker tags, skipping");
return;
}
let query = make_suspended_pull_query(&wc.worker_tags);
let mut l = WORKER_SUSPENDED_PULL_QUERY.write().await;
*l = query;
}
pub async fn make_pull_query(wc: &WorkerConfig) {
pub fn make_pull_query(tags: &[String]) -> String {
format_pull_query(format!(
"SELECT id
FROM v2_job_queue
WHERE running = false AND tag IN ({}) AND scheduled_for <= now()
ORDER BY priority DESC NULLS LAST, scheduled_for
FOR UPDATE SKIP LOCKED
LIMIT 1",
tags.iter().map(|x| format!("'{x}'")).join(", ")
))
}
pub async fn store_pull_query(wc: &WorkerConfig) {
let mut queries = vec![];
for tags in wc.priority_tags_sorted.iter() {
if tags.tags.len() == 0 {
tracing::error!("Empty tags in priority tags, skipping");
continue;
}
let query = format_pull_query(format!(
"SELECT id
FROM v2_job_queue
WHERE running = false AND tag IN ({}) AND scheduled_for <= now()
ORDER BY priority DESC NULLS LAST, scheduled_for
FOR UPDATE SKIP LOCKED
LIMIT 1",
tags.tags.iter().map(|x| format!("'{x}'")).join(", ")
));
let query = make_pull_query(&tags.tags);
queries.push(query);
}
let mut l = WORKER_PULL_QUERIES.write().await;
@@ -758,24 +886,33 @@ pub fn get_windmill_memory_usage() -> Option<i64> {
}
}
pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postgres>>(
executor: E,
) -> bool {
pub async fn update_min_version(conn: &Connection) -> bool {
use crate::utils::{GIT_SEM_VERSION, GIT_VERSION};
// fetch all pings with a different version than self from the last 5 minutes.
let pings = sqlx::query_scalar!(
"SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'",
GIT_VERSION
).fetch_all(executor).await.unwrap_or_default();
let cur_version = GIT_SEM_VERSION.clone();
let min_version = pings
.iter()
.filter(|x| !x.is_empty())
.filter_map(|x| semver::Version::parse(if x.starts_with('v') { &x[1..] } else { x }).ok())
.min()
.unwrap_or_else(|| cur_version.clone());
let min_version = match conn {
Connection::Sql(pool) => {
// fetch all pings with a different version than self from the last 5 minutes.
let pings = sqlx::query_scalar!(
"SELECT wm_version FROM worker_ping WHERE wm_version != $1 AND ping_at > now() - interval '5 minutes'",
GIT_VERSION
).fetch_all(pool).await.unwrap_or_default();
pings
.iter()
.filter(|x| !x.is_empty())
.filter_map(|x| {
semver::Version::parse(if x.starts_with('v') { &x[1..] } else { x }).ok()
})
.min()
.unwrap_or_else(|| cur_version.clone())
}
Connection::Http(_) => {
// TODO: get min version from server, for now we use the current version. Min version should be of no interest for http mode workers
cur_version.clone()
}
};
if min_version != cur_version {
tracing::info!("Minimal worker version: {min_version}");
@@ -790,37 +927,330 @@ pub async fn update_min_version<'c, E: sqlx::Executor<'c, Database = sqlx::Postg
min_version >= cur_version
}
pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) {
let (tags, dw) = {
let wc = WORKER_CONFIG.read().await.clone();
(
wc.worker_tags,
wc.dedicated_worker
.as_ref()
.map(|x| format!("{}:{}", x.workspace_id, x.path)),
)
};
#[derive(Serialize, Deserialize)]
pub enum PingType {
Initial,
MainLoop,
Job,
InitScript,
}
#[derive(Serialize, Deserialize)]
pub struct Ping {
pub last_job_executed: Option<Uuid>,
pub last_job_workspace_id: Option<String>,
pub worker_instance: Option<String>,
pub ip: Option<String>,
pub tags: Option<Vec<String>>,
pub dw: Option<String>,
pub version: Option<String>,
pub vcpus: Option<i64>,
pub memory: Option<i64>,
pub memory_usage: Option<i64>,
pub wm_memory_usage: Option<i64>,
pub jobs_executed: Option<i32>,
pub occupancy_rate: Option<f32>,
pub occupancy_rate_15s: Option<f32>,
pub occupancy_rate_5m: Option<f32>,
pub occupancy_rate_30m: Option<f32>,
pub ping_type: PingType,
}
pub async fn update_ping_http(
insert_ping: Ping,
worker_name: &str,
worker_group: &str,
db: &DB,
) -> anyhow::Result<()> {
// tracing::info!("update ping: {}", insert_ping.tags.join(","));
match insert_ping.ping_type {
PingType::MainLoop => {
update_worker_ping_main_loop_query(
worker_name,
insert_ping.tags.unwrap_or_default().as_slice(),
insert_ping.vcpus,
insert_ping.memory,
insert_ping.jobs_executed,
insert_ping.occupancy_rate,
insert_ping.memory_usage,
insert_ping.wm_memory_usage,
insert_ping.occupancy_rate_15s,
insert_ping.occupancy_rate_5m,
insert_ping.occupancy_rate_30m,
db,
)
.await?
}
PingType::Initial => {
if insert_ping.worker_instance.is_none()
|| insert_ping.version.is_none()
|| insert_ping.ip.is_none()
{
return Err(anyhow::anyhow!(
"Worker instance, version and ip are required"
));
}
let vcpus = get_vcpus();
let memory = get_memory();
insert_ping_query(
&insert_ping.worker_instance.unwrap(),
&worker_name,
worker_group,
&insert_ping.ip.unwrap(),
insert_ping.tags.unwrap_or_default().as_slice(),
insert_ping.dw,
&insert_ping.version.unwrap(),
insert_ping.vcpus,
insert_ping.memory,
db,
)
.await?;
}
PingType::Job => {
update_worker_ping_from_job_query(
&insert_ping.last_job_executed.unwrap_or_default(),
&insert_ping.last_job_workspace_id.unwrap_or_default(),
worker_name,
insert_ping.memory_usage,
insert_ping.wm_memory_usage,
insert_ping.occupancy_rate,
insert_ping.occupancy_rate_15s,
insert_ping.occupancy_rate_5m,
insert_ping.occupancy_rate_30m,
db,
)
.await?;
}
PingType::InitScript => {
update_ping_for_failed_init_script_query(
worker_name,
insert_ping.last_job_executed.unwrap_or_default(),
db,
)
.await?
}
}
Ok(())
}
#[derive(Serialize, Deserialize)]
pub struct JobCancelled {
pub canceled_by: String,
pub reason: String,
}
pub async fn set_job_cancelled_query(
job_id: Uuid,
db: &DB,
canceled_by: &str,
reason: &str,
) -> anyhow::Result<()> {
sqlx::query!(
"UPDATE v2_job_queue
SET canceled_by = $1
, canceled_reason = $2
WHERE id = $3",
canceled_by,
reason,
job_id
)
.execute(db)
.await?;
Ok(())
}
pub async fn update_ping_for_failed_init_script_query(
worker_name: &str,
last_job_id: Uuid,
db: &DB,
) -> anyhow::Result<()> {
sqlx::query!(
"UPDATE worker_ping SET
ping_at = now(),
jobs_executed = 1,
current_job_id = $1,
current_job_workspace_id = 'admins'
WHERE worker = $2",
last_job_id,
worker_name
)
.execute(db)
.await?;
Ok(())
}
pub async fn fetch_flow_node_query(
db: &DB,
id: i64,
loc: &'static Location<'_>,
) -> error::Result<RawNode> {
let r = sqlx::query!(
"SELECT \
code AS \"raw_code: String\", \
lock AS \"raw_lock: String\", \
flow AS \"raw_flow: Json<Box<RawValue>>\" \
FROM flow_node WHERE id = $1 LIMIT 1",
id,
)
.fetch_optional(db)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(loc, "Flow node", id))
.map(|r| RawNode {
raw_code: r.raw_code,
raw_lock: r.raw_lock,
raw_flow: r.raw_flow.map(|Json(raw_flow)| raw_flow),
})?;
Ok(r)
}
pub async fn fetch_raw_script_from_app_query(
db: &DB,
id: i64,
loc: &'static Location<'_>,
) -> error::Result<RawScript> {
sqlx::query!(
"SELECT lock, code FROM app_script WHERE id = $1 LIMIT 1",
id,
)
.fetch_optional(db)
.await
.map_err(Into::into)
.and_then(unwrap_or_error(&loc, "Application script", id))
.map(|r| RawScript { content: r.code, lock: r.lock, meta: None })
}
pub async fn insert_ping_query(
worker_instance: &str,
worker_name: &str,
worker_group: &str,
ip: &str,
tags: &[String],
dw: Option<String>,
version: &str,
vcpus: Option<i64>,
memory: Option<i64>,
db: &DB,
) -> anyhow::Result<()> {
sqlx::query!(
"INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5",
worker_instance,
worker_name,
ip,
tags.as_slice(),
*WORKER_GROUP,
tags,
worker_group,
dw,
crate::utils::GIT_VERSION,
version,
vcpus,
memory
)
.execute(db)
.await?;
Ok(())
}
pub async fn update_worker_ping_from_job_query(
job_id: &Uuid,
w_id: &str,
worker_name: &str,
memory_usage: Option<i64>,
wm_memory_usage: Option<i64>,
occupancy_rate: Option<f32>,
occupancy_rate_15s: Option<f32>,
occupancy_rate_5m: Option<f32>,
occupancy_rate_30m: Option<f32>,
db: &DB,
) -> anyhow::Result<()> {
sqlx::query!(
"UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,
occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
job_id,
w_id,
memory_usage,
wm_memory_usage,
worker_name,
occupancy_rate,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
)
.execute(db)
.await
.expect("insert worker_ping initial value");
.await?;
Ok(())
}
pub async fn update_job_ping_query(
job_id: &Uuid,
db: &DB,
mem_peak: Option<i32>,
) -> anyhow::Result<PingJobStatusResponse> {
let ro = sqlx::query!(
"UPDATE v2_job_runtime r SET
memory_peak = $1,
ping = now()
FROM v2_job_queue q
WHERE r.id = $2 AND q.id = r.id
RETURNING canceled_by, canceled_reason",
mem_peak,
job_id
)
.map(|x| PingJobStatusResponse {
canceled_by: x.canceled_by,
canceled_reason: x.canceled_reason,
already_completed: false,
})
.fetch_optional(db)
.await;
// TODO: add memory metrics to memory time series
if let Ok(r) = ro {
if let Some(i) = r {
Ok(i)
} else {
Err(anyhow::anyhow!("Job not found"))
}
} else {
Err(to_anyhow(ro.unwrap_err()))
}
}
pub async fn update_worker_ping_main_loop_query(
worker_name: &str,
tags: &[String],
vcpus: Option<i64>,
memory: Option<i64>,
jobs_executed: Option<i32>,
occupancy_rate: Option<f32>,
memory_usage: Option<i64>,
wm_memory_usage: Option<i64>,
occupancy_rate_15s: Option<f32>,
occupancy_rate_5m: Option<f32>,
occupancy_rate_30m: Option<f32>,
db: &DB,
) -> anyhow::Result<()> {
sqlx::query!(
"UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,
occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),
memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
jobs_executed,
tags,
occupancy_rate,
memory_usage,
wm_memory_usage,
worker_name,
vcpus,
memory,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
)
.execute(db)
.await?;
Ok(())
}
// "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,
// occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),
// memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
pub async fn load_worker_config(
db: &DB,
killpill_tx: KillpillSender,
@@ -1018,7 +1448,7 @@ pub struct WorkspacedPath {
pub path: String,
}
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, Debug)]
pub struct WorkerConfigOpt {
pub worker_tags: Option<Vec<String>>,
pub priority_tags: Option<HashMap<String, u8>>,
backend/windmill-queue/src/flow_status.rs
+135
View File
@@ -0,0 +1,135 @@
use uuid::Uuid;
use windmill_common::{
error::{self, Error},
utils::WarnAfterExt,
DB,
};
#[derive(Debug, Copy, Clone)]
pub enum Step {
Step(usize),
PreprocessorStep,
FailureStep,
}
impl Step {
pub fn from_i32_and_len(step: i32, len: usize) -> Self {
if step < 0 {
Step::PreprocessorStep
} else if (step as usize) < len {
Step::Step(step as usize)
} else {
Step::FailureStep
}
}
}
pub async fn update_flow_status_in_progress(
db: &DB,
_w_id: &str,
flow: Uuid,
job_in_progress: Uuid,
) -> error::Result<Step> {
let step = get_step_of_flow_status(db, flow).await?;
match step {
Step::Step(step) => {
sqlx::query!(
"UPDATE v2_job_status SET
flow_status = jsonb_set(
jsonb_set(flow_status, ARRAY['modules', $3::INTEGER::TEXT, 'job'], to_jsonb($1::UUID::TEXT)),
ARRAY['modules', $3::INTEGER::TEXT, 'type'],
to_jsonb('InProgress'::text)
)
WHERE id = $2",
job_in_progress,
flow,
step as i32
)
.execute(db)
.await?;
}
Step::PreprocessorStep => {
sqlx::query!(
"UPDATE v2_job_status SET
flow_status = jsonb_set(
jsonb_set(flow_status, ARRAY['preprocessor_module', 'job'], to_jsonb($1::UUID::TEXT)),
ARRAY['preprocessor_module', 'type'],
to_jsonb('InProgress'::text)
)
WHERE id = $2",
job_in_progress,
flow
)
.execute(db)
.await?;
}
Step::FailureStep => {
sqlx::query!(
"UPDATE v2_job_status SET
flow_status = jsonb_set(
jsonb_set(flow_status, ARRAY['failure_module', 'job'], to_jsonb($1::UUID::TEXT)),
ARRAY['failure_module', 'type'],
to_jsonb('InProgress'::text)
)
WHERE id = $2",
job_in_progress,
flow
)
.execute(db)
.await?;
}
}
Ok(step)
}
pub async fn update_workflow_as_code_status(
db: &DB,
id: &Uuid,
parent_job: &Uuid,
) -> error::Result<()> {
let _ = sqlx::query_scalar!(
"UPDATE v2_job_status SET
workflow_as_code_status = jsonb_set(
jsonb_set(
COALESCE(workflow_as_code_status, '{}'::jsonb),
array[$1],
COALESCE(workflow_as_code_status->$1, '{}'::jsonb)
),
array[$1, 'started_at'],
to_jsonb(now()::text)
)
WHERE id = $2",
id.to_string(),
parent_job
)
.execute(db)
.warn_after_seconds(5)
.await
.inspect_err(|e| {
tracing::error!(
"Could not update parent job `started_at` in workflow as code status: {}",
e
)
});
Ok(())
}
// TODO: merge as a CTE
#[tracing::instrument(level = "trace", skip_all)]
async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result<Step> {
let r = sqlx::query!(
"SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len
FROM v2_job_status WHERE id = $1",
id
)
.fetch_one(db)
.await
.map_err(|e| Error::internal_err(format!("fetching step flow status: {e:#}")))?;
if let Some(step) = r.step {
Ok(Step::from_i32_and_len(step, r.len.unwrap_or(0) as usize))
} else {
Err(Error::internal_err("step is null".to_string()))
}
}
backend/windmill-queue/src/jobs.rs
+402 -87
View File
@@ -7,7 +7,7 @@
*/
use std::fmt;
use std::{borrow::Borrow, collections::HashMap, sync::Arc, vec};
use std::{collections::HashMap, sync::Arc, vec};
use anyhow::Context;
use async_recursion::async_recursion;
@@ -29,7 +29,9 @@ use uuid::Uuid;
use windmill_audit::audit_ee::{audit_log, AuditAuthor};
use windmill_audit::ActionKind;
use windmill_common::auth::JobPerms;
use windmill_common::utils::now_from_db;
use windmill_common::worker::{Connection, SCRIPT_TOKEN_EXPIRY};
use windmill_common::{
auth::{fetch_authed_from_permissioned_as, permissioned_as_to_username},
cache::{self, FlowData},
@@ -49,7 +51,7 @@ use windmill_common::{
users::{SUPERADMIN_NOTIFICATION_EMAIL, SUPERADMIN_SECRET_EMAIL},
utils::{not_found_if_none, report_critical_error, StripPath, WarnAfterExt},
worker::{
to_raw_value, CLOUD_HOSTED, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES,
to_raw_value, CLOUD_HOSTED,
DISABLE_FLOW_SCRIPT, MIN_VERSION_IS_AT_LEAST_1_432, MIN_VERSION_IS_AT_LEAST_1_440, NO_LOGS,
WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY,
},
@@ -65,8 +67,10 @@ use windmill_common::BASE_URL;
#[cfg(feature = "cloud")]
use windmill_common::users::SUPERADMIN_SYNC_EMAIL;
use crate::flow_status::{update_flow_status_in_progress, update_workflow_as_code_status};
use crate::jobs_ee::update_concurrency_counter;
use crate::schedule::{get_schedule_opt, push_scheduled_job};
use crate::tags::per_workspace_tag;
#[cfg(feature = "prometheus")]
lazy_static::lazy_static! {
@@ -102,8 +106,6 @@ lazy_static::lazy_static! {
.build().unwrap();
pub static ref JOB_TOKEN: Option<String> = std::env::var("JOB_TOKEN").ok();
static ref JOB_ARGS_AUDIT_LOGS: bool = std::env::var("JOB_ARGS_AUDIT_LOGS")
.ok()
.and_then(|x| x.parse().ok())
@@ -125,12 +127,26 @@ const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill
#[cfg(any(feature = "enterprise", feature = "cloud"))]
const SCHEDULE_RECOVERY_HANDLER_USER_EMAIL: &str = "schedule_recovery_handler@windmill.dev";
#[derive(Clone, Debug)]
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct CanceledBy {
pub username: Option<String>,
pub reason: Option<String>,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct JobCompleted {
pub job: Arc<MiniPulledJob>,
pub result: Arc<Box<RawValue>>,
pub result_columns: Option<Vec<String>>,
pub mem_peak: i32,
pub success: bool,
pub cached_res_path: Option<String>,
pub token: String,
pub canceled_by: Option<CanceledBy>,
pub duration: Option<i64>,
}
pub async fn cancel_single_job<'c>(
username: &str,
reason: Option<String>,
@@ -144,17 +160,19 @@ pub async fn cancel_single_job<'c>(
let username = username.to_string();
let w_id = w_id.to_string();
let db = db.clone();
tracing::info!("cancelling job {:?}", db);
let job_running = job_running.clone();
tokio::task::spawn(async move {
let reason: String = reason
.clone()
.unwrap_or_else(|| "unexplicited reasons".to_string());
let e = serde_json::json!({"message": format!("Job canceled: {reason} by {username}"), "name": "Canceled", "reason": reason, "canceler": username});
append_logs(
&job_running.id,
w_id.to_string(),
format!("canceled by {username}: (force cancel: {force_cancel})"),
&db,
&Connection::from(db.clone()),
)
.await;
let add_job = add_completed_job_error(
@@ -335,7 +353,7 @@ pub async fn append_logs(
job_id: &uuid::Uuid,
workspace: impl AsRef<str>,
logs: impl AsRef<str>,
db: impl Borrow<Pool<Postgres>>,
conn: &Connection,
) {
if logs.as_ref().is_empty() {
return;
@@ -350,20 +368,81 @@ pub async fn append_logs(
tracing::info!("NO LOGS [{job_id}]: {}", logs.as_ref());
return;
}
if let Err(err) = sqlx::query!(
"INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text)",
logs.as_ref(),
job_id,
workspace.as_ref(),
)
.execute(db.borrow())
.warn_after_seconds(1)
.await
{
tracing::error!(%job_id, %err, "error updating logs for large_log job {job_id}: {err}");
match conn {
Connection::Sql(pool) => {
if let Err(err) = sqlx::query!(
"INSERT INTO job_logs (logs, job_id, workspace_id) VALUES ($1, $2, $3) ON CONFLICT (job_id) DO UPDATE SET logs = concat(job_logs.logs, $1::text)",
logs.as_ref(),
job_id,
workspace.as_ref(),
)
.execute(pool)
.warn_after_seconds(1)
.await
{
tracing::error!(%job_id, %err, "error updating logs for job {job_id}: {err}");
}
}
Connection::Http(client) => {
if let Err(e) = client
.post::<_, String>(
&format!("/api/w/{}/agent_workers/push_logs/{}", workspace.as_ref(), job_id),
&logs.as_ref(),
)
.await {
tracing::error!(%job_id, %e, "error sending logs for job {job_id}: {e}");
};
}
}
}
pub async fn push_init_job<'c>(
db: &Pool<Postgres>,
content: String,
worker_name: &str,
) -> error::Result<Uuid> {
let tx = PushIsolationLevel::IsolatedRoot(db.clone());
let ehm = HashMap::new();
let (uuid, inner_tx) = push(
&db,
tx,
"admins",
windmill_common::jobs::JobPayload::Code(windmill_common::jobs::RawCode {
hash: None,
content,
path: Some(format!("init_script_{worker_name}")),
language: ScriptLang::Bash,
lock: None,
custom_concurrency_key: None,
concurrent_limit: None,
concurrency_time_window_s: None,
cache_ttl: None,
dedicated_worker: None,
}),
PushArgs::from(&ehm),
worker_name,
"worker@windmill.dev",
SUPERADMIN_SECRET_EMAIL.to_string(),
None,
None,
None,
None,
None,
false,
true,
None,
true,
Some("init_script".to_string()),
None,
None,
None,
None,
)
.await?;
inner_tx.commit().await?;
Ok(uuid)
}
pub async fn cancel_persistent_script_jobs<'c>(
username: &str,
reason: Option<String>,
@@ -877,11 +956,10 @@ pub async fn add_completed_job<T: Serialize + Send + Sync + ValidableJson>(
tracing::debug!("decremented concurrency counter");
}
if JOB_TOKEN.is_none() {
sqlx::query!("DELETE FROM job_perms WHERE job_id = $1", job_id)
.execute(&mut *tx)
.await?;
}
tx.commit().await?;
@@ -1909,7 +1987,7 @@ impl fmt::Display for TriggerKind {
}
}
#[derive(sqlx::FromRow, Debug, Clone)]
#[derive(sqlx::FromRow, Debug, Clone, Serialize, Deserialize)]
pub struct MiniPulledJob {
pub workspace_id: String,
pub id: Uuid,
@@ -2011,6 +2089,10 @@ impl MiniPulledJob {
self.kind.is_flow()
}
pub fn is_dependency(&self) -> bool {
self.kind.is_dependency()
}
pub fn schedule_path(&self) -> Option<String> {
if self
.trigger_kind
@@ -2022,9 +2104,35 @@ impl MiniPulledJob {
None
}
}
pub async fn mark_as_started_if_step(&self, db: &DB) -> Result<(), Error> {
if self.is_flow_step() {
let _ = update_flow_status_in_progress(
db,
&self.workspace_id,
self.parent_job
.ok_or_else(|| Error::internal_err(format!("expected parent job")))?,
self.id,
)
.warn_after_seconds(5)
.await?;
} else if let Some(parent_job) = self.parent_job {
let _ = update_workflow_as_code_status(
db,
&self.id,
&parent_job,
)
.await?;
}
Ok(())
}
}
#[derive(sqlx::FromRow, Debug, Clone)]
#[derive(sqlx::FromRow, Debug, Clone, Serialize, Deserialize)]
pub struct PulledJob {
#[sqlx(flatten)]
pub job: MiniPulledJob,
@@ -2040,6 +2148,95 @@ pub struct PulledJob {
pub permissioned_as_folders: Option<Vec<serde_json::Value>>,
}
#[derive(Serialize, Deserialize)]
pub enum PrecomputedAgentInfo {
Bun { local: String, remote: String },
Python { py_version: Option<u32>, requirements: Option<String> },
}
#[derive(Serialize, Deserialize)]
pub struct JobAndPerms {
pub job: MiniPulledJob,
pub raw_code: Option<String>,
pub raw_flow: Option<Json<Box<RawValue>>>,
pub raw_lock: Option<String>,
pub parent_runnable_path: Option<String>,
pub token: String,
pub precomputed_agent_info: Option<PrecomputedAgentInfo>,
}
impl PulledJob {
pub async fn get_job_and_perms(self, db: &DB) -> JobAndPerms {
let job_perms = match (
self.permissioned_as_email,
self.permissioned_as_username,
self.permissioned_as_is_admin,
self.permissioned_as_is_operator,
self.permissioned_as_groups,
self.permissioned_as_folders,
) {
(
Some(email),
Some(username),
Some(is_admin),
Some(is_operator),
Some(groups),
Some(folders),
) => Some(JobPerms {
email,
username,
is_admin,
is_operator,
groups,
folders,
}),
_ => None,
};
let token = create_token(&db, &self.job, job_perms).await;
JobAndPerms {
job: self.job,
raw_code: self.raw_code,
raw_flow: self.raw_flow,
raw_lock: self.raw_lock,
parent_runnable_path: self.parent_runnable_path,
token,
precomputed_agent_info: None,
}
}
}
// struct Permission
pub async fn create_token(db: &DB, job: &MiniPulledJob, perms: Option<JobPerms>) -> String {
// skipping test runs
if job.workspace_id != "" {
let label = if job.permissioned_as != format!("u/{}", job.created_by)
&& job.permissioned_as != job.created_by
{
format!("ephemeral-script-end-user-{}", job.created_by)
} else {
"ephemeral-script".to_string()
};
windmill_common::auth::create_token_for_owner(
db,
&job.workspace_id,
&job.permissioned_as,
&label,
*SCRIPT_TOKEN_EXPIRY,
&job.permissioned_as_email,
&job.id,
perms,
)
.warn_after_seconds(5)
.await
.expect("could not create job token")
} else {
return "".to_string();
}
}
impl std::ops::Deref for PulledJob {
type Target = MiniPulledJob;
fn deref(&self) -> &Self::Target {
@@ -2097,23 +2294,68 @@ pub async fn get_mini_pulled_job<'c>(
Ok(job)
}
#[derive(Serialize, Deserialize, Debug)]
pub struct PulledJobResult {
pub job: Option<PulledJob>,
pub suspended: bool,
}
pub async fn pull(
db: &Pool<Postgres>,
suspend_first: bool,
worker_name: &str,
) -> windmill_common::error::Result<(Option<PulledJob>, bool)> {
query_o: Option<(String, String)>,
) -> windmill_common::error::Result<PulledJobResult> {
loop {
if let Some((query_suspended, query_no_suspend)) = query_o.as_ref() {
let njob = {
let job = sqlx::query_as::<_, PulledJob>(query_suspended)
.bind(worker_name)
.fetch_optional(db)
.await?;
if let Some(job) = job {
PulledJobResult { job: Some(job), suspended: true }
} else {
let job = sqlx::query_as::<_, PulledJob>(query_no_suspend)
.bind(worker_name)
.fetch_optional(db)
.await?;
PulledJobResult { job, suspended: false }
}
};
if let Some(job) = njob.job.as_ref() {
if job.is_flow() || job.is_dependency() {
let per_workspace = per_workspace_tag(&job.workspace_id).await;
let base_tag = if job.is_flow() {
"flow".to_string()
} else {
"dependency".to_string()
};
let tag = if per_workspace {
format!("{}-{}", base_tag, job.workspace_id)
} else {
base_tag
};
sqlx::query!("UPDATE v2_job_queue SET tag = $1, running = false WHERE id = $2", tag, job.id).execute(db).await?;
continue;
}
}
return Ok(njob);
};
let (job, suspended) = pull_single_job_and_mark_as_running_no_concurrency_limit(
db,
suspend_first,
worker_name,
)
.await?;
db,
suspend_first,
worker_name,
)
.await?;
let Some(job) = job else {
return Ok((None, suspended));
return Ok(PulledJobResult { job: None, suspended });
};
let has_concurent_limit = job.concurrent_limit.is_some();
#[cfg(not(feature = "enterprise"))]
@@ -2134,7 +2376,7 @@ pub async fn pull(
if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
QUEUE_PULL_COUNT.inc();
}
return Ok((Option::Some(pulled_job), suspended));
return Ok(PulledJobResult { job: Some(pulled_job), suspended });
}
let job_concurrency_key = match concurrency_key(db, &pulled_job.id).await {
@@ -2184,7 +2426,7 @@ pub async fn pull(
if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
QUEUE_PULL_COUNT.inc();
}
return Ok((Option::Some(pulled_job), suspended));
return Ok(PulledJobResult { job: Some(pulled_job), suspended });
}
let job_script_path = pulled_job.runnable_path.clone().unwrap_or_default();
@@ -2273,7 +2515,13 @@ pub async fn pull(
"\nRe-scheduled job to {estimated_next_schedule_timestamp} due to concurrency limits with key {job_concurrency_key} and limit {job_custom_concurrent_limit} in the last {job_custom_concurrency_time_window_s} seconds (min_started_at: {min_started_at_or_now}, avg script duration: {:?}, number of time windows full: {})\n",
avg_script_duration, i
);
let _ = append_logs(&job_uuid, &pulled_job.workspace_id, job_log_event, db).await;
let _ = append_logs(
&job_uuid,
&pulled_job.workspace_id,
job_log_event,
&Connection::from(db.clone()),
)
.await;
sqlx::query!(
"
@@ -3811,14 +4059,6 @@ pub async fn push<'c, 'd>(
.map(|e| (Some(e.0), e.1))
.unwrap_or_else(|| (None, None));
let per_workspace_workspaces = DEFAULT_TAGS_WORKSPACES.read().await;
let per_workspace = DEFAULT_TAGS_PER_WORKSPACE.load(std::sync::atomic::Ordering::Relaxed)
&& (per_workspace_workspaces.is_none()
|| per_workspace_workspaces
.as_ref()
.unwrap()
.contains(&workspace_id.to_string()));
let tag = if dedicated_worker.is_some_and(|x| x) {
format!(
"{}:{}{}",
@@ -3836,6 +4076,7 @@ pub async fn push<'c, 'd>(
}
let interpolated_tag = tag.map(|x| interpolate_args(x, &args, workspace_id));
let per_workspace = per_workspace_tag(&workspace_id).await;
let default = || {
let ntag = if job_kind.is_flow() || job_kind == JobKind::Identity {
@@ -4019,54 +4260,53 @@ pub async fn push<'c, 'd>(
QUEUE_PUSH_COUNT.inc();
}
if JOB_TOKEN.is_none() {
let job_authed = match authed {
Some(authed)
if authed.email == email
&& authed.username == permissioned_as_to_username(&permissioned_as) =>
{
authed.clone()
}
_ => {
if authed.is_some() {
tracing::warn!("Authed passed to push is not the same as permissioned_as, refetching directly permissions for job {job_id}...")
}
fetch_authed_from_permissioned_as(
permissioned_as.clone(),
email.to_string(),
workspace_id,
_db,
)
.await
.map_err(|e| {
Error::internal_err(format!(
"Could not get permissions directly for job {job_id}: {e:#}"
))
})?
}
};
let folders = job_authed
.folders
.iter()
.filter_map(|x| serde_json::to_value(x).ok())
.collect::<Vec<_>>();
if let Err(err) = sqlx::query!("INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id)
values ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8",
job_id,
job_authed.email,
job_authed.username,
job_authed.is_admin,
job_authed.is_operator,
folders.as_slice(),
job_authed.groups.as_slice(),
workspace_id,
).execute(&mut *tx).await {
tracing::error!("Could not insert job_perms for job {job_id}: {err:#}");
let job_authed = match authed {
Some(authed)
if authed.email == email
&& authed.username == permissioned_as_to_username(&permissioned_as) =>
{
authed.clone()
}
_ => {
if authed.is_some() {
tracing::warn!("Authed passed to push is not the same as permissioned_as, refetching direclty permissions for job {job_id}...")
}
fetch_authed_from_permissioned_as(
permissioned_as.clone(),
email.to_string(),
workspace_id,
_db,
)
.await
.map_err(|e| {
Error::internal_err(format!(
"Could not get permissions directly for job {job_id}: {e:#}"
))
})?
}
};
let folders = job_authed
.folders
.iter()
.filter_map(|x| serde_json::to_value(x).ok())
.collect::<Vec<_>>();
if let Err(err) = sqlx::query!("INSERT INTO job_perms (job_id, email, username, is_admin, is_operator, folders, groups, workspace_id)
values ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (job_id) DO UPDATE SET email = $2, username = $3, is_admin = $4, is_operator = $5, folders = $6, groups = $7, workspace_id = $8",
job_id,
job_authed.email,
job_authed.username,
job_authed.is_admin,
job_authed.is_operator,
folders.as_slice(),
job_authed.groups.as_slice(),
workspace_id,
).execute(&mut *tx).await {
tracing::error!("Could not insert job_perms for job {job_id}: {err:#}");
}
{
let uuid_string = job_id.to_string();
@@ -4177,7 +4417,7 @@ async fn restarted_flows_resolution(
})?;
let flow_data = cache::job::fetch_flow(db, row.job_kind, row.script_hash)
.or_else(|_| cache::job::fetch_preview_flow(db, &completed_flow_id, row.raw_flow))
.or_else(|_| cache::job::fetch_preview_flow(db.into(), &completed_flow_id, row.raw_flow))
.await?;
let flow_value = flow_data.value();
let flow_status = row
@@ -4330,3 +4570,78 @@ async fn restarted_flows_resolution(
flow_status.cleanup_module,
))
}
#[derive(Serialize, Deserialize)]
pub struct SameWorkerPayload {
pub job_id: Uuid,
pub recoverable: bool,
}
pub async fn get_same_worker_job(
db: &DB,
same_worker_job: &SameWorkerPayload,
) -> windmill_common::error::Result<Option<PulledJob>> {
sqlx::query_as::<_, PulledJob>(
"WITH ping AS (
UPDATE v2_job_runtime SET ping = NOW() WHERE id = $1
),
started_at AS (
UPDATE v2_job_queue SET started_at = NOW() WHERE id = $1
)
SELECT
v2_job_queue.workspace_id,
v2_job_queue.id,
v2_job.args,
v2_job.parent_job,
v2_job.created_by,
v2_job_queue.started_at,
scheduled_for,
v2_job.runnable_path,
v2_job.kind,
v2_job.runnable_id,
v2_job_queue.canceled_reason,
v2_job_queue.canceled_by,
v2_job.permissioned_as,
v2_job.permissioned_as_email,
v2_job_status.flow_status,
v2_job.tag,
v2_job.script_lang,
v2_job.same_worker,
v2_job.pre_run_error,
v2_job.concurrent_limit,
v2_job.concurrency_time_window_s,
v2_job.flow_innermost_root_job,
v2_job.timeout,
v2_job.flow_step_id,
v2_job.cache_ttl,
v2_job_queue.priority,
v2_job.preprocessed,
v2_job.script_entrypoint_override,
v2_job.trigger,
v2_job.trigger_kind,
v2_job.visible_to_owner,
v2_job.raw_code,
v2_job.raw_lock,
v2_job.raw_flow,
pj.runnable_path as parent_runnable_path,
p.email as permissioned_as_email, p.username as permissioned_as_username, p.is_admin as permissioned_as_is_admin,
p.is_operator as permissioned_as_is_operator, p.groups as permissioned_as_groups, p.folders as permissioned_as_folders
FROM v2_job_queue
INNER JOIN v2_job ON v2_job.id = v2_job_queue.id
LEFT JOIN v2_job_status ON v2_job_status.id = v2_job_queue.id
LEFT JOIN job_perms p ON p.job_id = v2_job.id
LEFT JOIN v2_job pj ON v2_job.parent_job = pj.id
WHERE v2_job_queue.id = $1
",
)
.bind(same_worker_job.job_id)
.fetch_optional(db)
.await
.map_err(|e| {
Error::internal_err(format!(
"Impossible to fetch same_worker job {}: {}",
same_worker_job.job_id, e
))
})
}
backend/windmill-queue/src/lib.rs
+2
View File
@@ -10,3 +10,5 @@ mod jobs;
pub mod jobs_ee;
pub mod schedule;
pub use jobs::*;
pub mod flow_status;
pub mod tags;
backend/windmill-queue/src/tags.rs
+11
View File
@@ -0,0 +1,11 @@
use windmill_common::worker::{DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES};
pub async fn per_workspace_tag(workspace_id: &str) -> bool {
let per_workspace_workspaces = DEFAULT_TAGS_WORKSPACES.read().await;
DEFAULT_TAGS_PER_WORKSPACE.load(std::sync::atomic::Ordering::Relaxed)
&& (per_workspace_workspaces.is_none()
|| per_workspace_workspaces
.as_ref()
.unwrap()
.contains(&workspace_id.to_string()))
}
backend/windmill-worker/Cargo.toml
+1
View File
@@ -103,6 +103,7 @@ urlencoding.workspace = true
nix.workspace = true
bytes.workspace = true
reqwest.workspace = true
reqwest-middleware.workspace = true
hex.workspace = true
tiberius = { workspace = true, optional = true }
tokio-util = { workspace = true, optional = true }
backend/windmill-worker/src/agent_workers.rs
+31
View File
@@ -0,0 +1,31 @@
use uuid::Uuid;
use windmill_common::{agent_workers::QueueInitJob, worker::HttpClient};
use windmill_queue::{JobAndPerms, JobCompleted};
pub async fn queue_init_job(client: &HttpClient, content: &str) -> anyhow::Result<Uuid> {
client
.post(
"/api/agent_workers/queue_init_job",
&QueueInitJob { content: content.to_string() },
)
.await
.and_then(|x: String| Uuid::parse_str(&x).map_err(|e| anyhow::anyhow!(e)))
}
pub async fn pull_job(client: &HttpClient) -> anyhow::Result<Option<JobAndPerms>> {
client.post("/api/agent_workers/pull_job", &()).await
}
pub async fn send_result(client: &HttpClient, jc: JobCompleted) -> anyhow::Result<String> {
client
.post(
&format!(
"/api/w/{}/agent_workers/send_result/{}",
jc.job.workspace_id, jc.job.id
),
&jc,
)
.await
}
pub const UPDATE_PING_URL: &str = "/api/agent_workers/update_ping";
backend/windmill-worker/src/ansible_executor.rs
+30 -26
View File
@@ -11,7 +11,9 @@ use tokio::process::Command;
use uuid::Uuid;
use windmill_common::{
error,
worker::{to_raw_value, write_file, write_file_at_user_defined_location, WORKER_CONFIG},
worker::{
to_raw_value, write_file, write_file_at_user_defined_location, Connection, WORKER_CONFIG,
},
};
use windmill_queue::MiniPulledJob;
@@ -46,7 +48,7 @@ async fn handle_ansible_python_deps(
ansible_reqs: Option<&AnsibleRequirements>,
w_id: &str,
job_id: &Uuid,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
worker_name: &str,
worker_dir: &str,
mem_peak: &mut i32,
@@ -77,7 +79,7 @@ async fn handle_ansible_python_deps(
mem_peak,
canceled_by,
job_dir,
db,
conn,
worker_name,
w_id,
&mut Some(occupancy_metrics),
@@ -103,7 +105,7 @@ async fn handle_ansible_python_deps(
w_id,
mem_peak,
canceled_by,
db,
conn,
worker_name,
job_dir,
worker_dir,
@@ -124,7 +126,7 @@ async fn install_galaxy_collections(
w_id: &str,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
occupancy_metrics: &mut OccupancyMetrics,
) -> anyhow::Result<()> {
write_file(job_dir, "requirements.yml", collections_yml)?;
@@ -133,7 +135,7 @@ async fn install_galaxy_collections(
job_id,
w_id,
"\n\n--- ANSIBLE GALAXY INSTALL ---\n".to_string(),
db,
conn,
)
.await;
let mut galaxy_command = Command::new(ANSIBLE_GALAXY_PATH.as_str());
@@ -159,7 +161,7 @@ async fn install_galaxy_collections(
let child = start_child_process(galaxy_command, ANSIBLE_GALAXY_PATH.as_str()).await?;
handle_child(
job_id,
db,
conn,
mem_peak,
canceled_by,
child,
@@ -185,7 +187,7 @@ pub async fn handle_ansible_job(
job: &MiniPulledJob,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
client: &AuthedClient,
parent_runnable_path: Option<String>,
inner_content: &String,
@@ -201,7 +203,7 @@ pub async fn handle_ansible_job(
)?;
let (logs, reqs, playbook) = windmill_parser_yaml::parse_ansible_reqs(inner_content)?;
append_logs(&job.id, &job.workspace_id, logs, db).await;
append_logs(&job.id, &job.workspace_id, logs, conn).await;
write_file(job_dir, "main.yml", &playbook)?;
let additional_python_paths = handle_ansible_python_deps(
@@ -210,7 +212,7 @@ pub async fn handle_ansible_job(
reqs.as_ref(),
&job.workspace_id,
&job.id,
db,
conn,
worker_name,
worker_dir,
mem_peak,
@@ -227,7 +229,7 @@ pub async fn handle_ansible_job(
args.insert(name.clone(), to_raw_value(path));
}
}
if let Some(x) = transform_json(client, &job.workspace_id, &args, job, db).await? {
if let Some(x) = transform_json(client, &job.workspace_id, &args, job, conn).await? {
write_file(
job_dir,
"args.json",
@@ -267,16 +269,18 @@ pub async fn handle_ansible_job(
let mut nsjail_extra_mounts = vec![];
if let Some(r) = reqs {
nsjail_extra_mounts = create_file_resources(
&job.id,
&job.workspace_id,
job_dir,
interpolated_args.as_ref(),
&r,
&client,
db,
)
.await?;
if let Some(db) = conn.as_sql() {
nsjail_extra_mounts = create_file_resources(
&job.id,
&job.workspace_id,
job_dir,
interpolated_args.as_ref(),
&r,
&client,
db,
)
.await?;
}
if let Some(collections) = r.collections {
install_galaxy_collections(
@@ -287,7 +291,7 @@ pub async fn handle_ansible_job(
&job.workspace_id,
mem_peak,
canceled_by,
db,
conn,
occupancy_metrics,
)
.await?;
@@ -297,7 +301,7 @@ pub async fn handle_ansible_job(
&job.id,
&job.workspace_id,
"\n\n--- ANSIBLE PLAYBOOK EXECUTION ---\n".to_string(),
db,
conn,
)
.await;
let ansible_cfg_content = format!(
@@ -313,7 +317,7 @@ remote_tmp={job_dir}/.ansible/tmp
write_file(job_dir, "ansible.cfg", &ansible_cfg_content)?;
let mut reserved_variables =
get_reserved_variables(job, &client.token, db, parent_runnable_path).await?;
get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
let additional_python_paths_folders = additional_python_paths.join(":");
if !*DISABLE_NSJAIL {
@@ -422,7 +426,7 @@ fi
handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
@@ -564,7 +568,7 @@ async fn create_file_resources(
file_res.target_path, file_res.resource_path
));
}
append_logs(job_id, w_id, logs, db).await;
append_logs(job_id, w_id, logs, &Connection::Sql(db.clone())).await;
Ok(nsjail_mounts)
}
backend/windmill-worker/src/bash_executor.rs
+13 -16
View File
@@ -15,12 +15,9 @@ use tokio::process::Command;
use uuid::Uuid;
use windmill_common::{
error::Error,
worker::{to_raw_value, write_file},
worker::{to_raw_value, write_file, Connection},
};
#[cfg(feature = "dind")]
use windmill_common::DB;
#[cfg(feature = "dind")]
use windmill_common::error::to_anyhow;
@@ -63,7 +60,7 @@ pub async fn handle_bash_job(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job: &MiniPulledJob,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
client: &AuthedClient,
parent_runnable_path: Option<String>,
content: &str,
@@ -81,7 +78,7 @@ pub async fn handle_bash_job(
if annotation.docker {
logs1.push_str("docker mode\n");
}
append_logs(&job.id, &job.workspace_id, logs1, db).await;
append_logs(&job.id, &job.workspace_id, logs1, &conn).await;
write_file(job_dir, "main.sh", &format!("set -e\n{content}"))?;
let script = format!(
@@ -137,10 +134,10 @@ exit $exit_status
write_file(job_dir, "wrapper.sh", &script)?;
let mut reserved_variables =
get_reserved_variables(job, &client.token, db, parent_runnable_path).await?;
get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
reserved_variables.insert("RUST_LOG".to_string(), "info".to_string());
let args = build_args_map(job, client, db).await?.map(Json);
let args = build_args_map(job, client, conn).await?.map(Json);
let job_args = if args.is_some() {
args.as_ref()
} else {
@@ -215,7 +212,7 @@ exit $exit_status
};
handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
@@ -235,7 +232,7 @@ exit $exit_status
return handle_docker_job(
job.id,
&job.workspace_id,
db,
conn,
job.timeout,
mem_peak,
canceled_by,
@@ -280,7 +277,7 @@ exit $exit_status
async fn handle_docker_job(
job_id: Uuid,
workspace_id: &str,
db: &DB,
conn: &Connection,
job_timeout: Option<i32>,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
@@ -315,7 +312,7 @@ async fn handle_docker_job(
let ncontainer_id = container_id.to_string();
let w_id = workspace_id.to_string();
let j_id = job_id.clone();
let db2 = db.clone();
let conn2 = conn.clone();
let (tx, mut rx) = tokio::sync::broadcast::channel::<()>(1);
let mut killpill_rx = killpill_rx.resubscribe();
@@ -337,7 +334,7 @@ async fn handle_docker_job(
log = log_stream.next() => {
match log {
Some(Ok(log)) => {
append_logs(&j_id, w_id.clone(), log.to_string(), db2.clone()).await;
append_logs(&j_id, w_id.clone(), log.to_string(), &conn2).await;
}
Some(Err(e)) => {
tracing::error!("Error getting logs: {:?}", e);
@@ -371,7 +368,7 @@ async fn handle_docker_job(
let result = run_future_with_polling_update_job_poller(
job_id,
job_timeout,
db,
conn,
mem_peak,
canceled_by,
wait_f,
@@ -472,7 +469,7 @@ pub async fn handle_powershell_job(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job: &MiniPulledJob,
db: &sqlx::Pool<sqlx::Postgres>,
db: &Connection,
client: &AuthedClient,
parent_runnable_path: Option<String>,
content: &str,
@@ -484,7 +481,7 @@ pub async fn handle_powershell_job(
occupancy_metrics: &mut OccupancyMetrics,
) -> Result<Box<RawValue>, Error> {
let pwsh_args = {
let args = build_args_map(job, client, db).await?.map(Json);
let args = build_args_map(job, client, &db).await?.map(Json);
let job_args = if args.is_some() {
args.as_ref()
} else {
backend/windmill-worker/src/bigquery_executor.rs
+5 -4
View File
@@ -5,6 +5,7 @@ use futures::{FutureExt, TryFutureExt};
use reqwest::Client;
use serde_json::{json, value::RawValue, Value};
use windmill_common::error::to_anyhow;
use windmill_common::worker::Connection;
use windmill_common::{error::Error, worker::to_raw_value};
use windmill_parser_sql::{
parse_bigquery_sig, parse_db_resource, parse_sql_blocks, parse_sql_statement_named_params,
@@ -209,14 +210,14 @@ pub async fn do_bigquery(
job: &MiniPulledJob,
client: &AuthedClient,
query: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
column_order: &mut Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> windmill_common::error::Result<Box<RawValue>> {
let bigquery_args = build_args_values(job, client, db).await?;
let bigquery_args = build_args_values(job, client, conn).await?;
let inline_db_res_path = parse_db_resource(&query);
@@ -252,7 +253,7 @@ pub async fn do_bigquery(
.map_err(|e| Error::ExecutionErr(e.to_string()))?;
let (timeout_duration, _, _) =
resolve_job_timeout(&db, &job.workspace_id, job.id, job.timeout).await;
resolve_job_timeout(&conn, &job.workspace_id, job.id, job.timeout).await;
let timeout_ms = timeout_duration.as_millis() as u64;
let http_client = build_http_client(timeout_duration)?;
@@ -366,7 +367,7 @@ pub async fn do_bigquery(
let r = run_future_with_polling_update_job_poller(
job.id,
job.timeout,
db,
conn,
mem_peak,
canceled_by,
result_f.map_err(to_anyhow),
backend/windmill-worker/src/bun_executor.rs
+54 -43
View File
@@ -9,7 +9,7 @@ use serde_json::value::RawValue;
use uuid::Uuid;
use windmill_parser_ts::remove_pinned_imports;
use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
use windmill_queue::{append_logs, CanceledBy, MiniPulledJob, PrecomputedAgentInfo};
#[cfg(feature = "enterprise")]
use crate::common::build_envs_map;
@@ -42,7 +42,7 @@ use windmill_common::{
error::{self, Result},
get_latest_hash_for_path,
scripts::ScriptLang,
worker::{exists_in_cache, save_cache, write_file, DISABLE_BUNDLING},
worker::{exists_in_cache, save_cache, write_file, Connection, DISABLE_BUNDLING},
DB,
};
@@ -96,7 +96,7 @@ pub async fn gen_bun_lockfile(
canceled_by: &mut Option<CanceledBy>,
job_id: &Uuid,
w_id: &str,
db: Option<&sqlx::Pool<sqlx::Postgres>>,
db: Option<&Connection>,
token: &str,
script_path: &str,
job_dir: &str,
@@ -273,7 +273,7 @@ pub async fn install_bun_lockfile(
canceled_by: &mut Option<CanceledBy>,
job_id: &Uuid,
w_id: &str,
db: Option<&sqlx::Pool<sqlx::Postgres>>,
db: Option<&Connection>,
job_dir: &str,
worker_name: &str,
common_bun_proc_envs: HashMap<String, String>,
@@ -488,7 +488,7 @@ pub async fn generate_wrapper_mjs(
w_id: &str,
job_id: &Uuid,
worker_name: &str,
db: &sqlx::Pool<sqlx::Postgres>,
db: &Connection,
timeout: Option<i32>,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
@@ -538,7 +538,7 @@ pub async fn generate_bun_bundle(
w_id: &str,
job_id: &Uuid,
worker_name: &str,
db: Option<sqlx::Pool<sqlx::Postgres>>,
db: Option<&Connection>,
timeout: Option<i32>,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
@@ -679,21 +679,15 @@ pub async fn prebundle_bun_script(
script_path: &str,
job_id: &Uuid,
w_id: &str,
db: Option<DB>,
db: Option<&DB>,
job_dir: &str,
base_internal_url: &str,
worker_name: &str,
token: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> Result<()> {
let (local_path, remote_path) = compute_bundle_local_and_remote_path(
inner_content,
lockfile,
script_path,
db.clone(),
w_id,
)
.await;
let (local_path, remote_path) =
compute_bundle_local_and_remote_path(inner_content, lockfile, script_path, db, w_id).await;
if exists_in_cache(&local_path, &remote_path).await {
return Ok(());
}
@@ -727,7 +721,7 @@ pub async fn prebundle_bun_script(
w_id,
job_id,
worker_name,
db.clone(),
db.map(|x| Connection::from(x.clone())).as_ref(),
None,
&mut 0,
&mut None,
@@ -755,11 +749,11 @@ async fn get_script_import_updated_at(db: &DB, w_id: &str, script_path: &str) ->
Ok(last_updated_at.to_string())
}
async fn compute_bundle_local_and_remote_path(
pub async fn compute_bundle_local_and_remote_path(
inner_content: &str,
requirements_o: Option<&String>,
script_path: &str,
db: Option<DB>,
db: Option<&DB>,
w_id: &str,
) -> (String, String) {
let mut input_src = format!(
@@ -827,7 +821,7 @@ pub async fn handle_bun_job(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job: &MiniPulledJob,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
client: &AuthedClient,
parent_runnable_path: Option<String>,
job_dir: &str,
@@ -838,6 +832,7 @@ pub async fn handle_bun_job(
shared_mount: &str,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
precomputed_agent_info: Option<PrecomputedAgentInfo>,
) -> error::Result<Box<RawValue>> {
let mut annotation = windmill_common::worker::TypeScriptAnnotations::parse(inner_content);
@@ -846,14 +841,29 @@ pub async fn handle_bun_job(
&& !*DISABLE_BUNDLING
&& codebase.is_none()
{
let (local_path, remote_path) = compute_bundle_local_and_remote_path(
inner_content,
requirements_o,
job.runnable_path(),
Some(db.clone()),
&job.workspace_id,
)
.await;
let (local_path, remote_path) = match conn {
Connection::Sql(db) => {
compute_bundle_local_and_remote_path(
inner_content,
requirements_o,
job.runnable_path(),
Some(db),
&job.workspace_id,
)
.await
}
Connection::Http(_) => {
let (local_path, remote_path) = match precomputed_agent_info {
Some(PrecomputedAgentInfo::Bun { local, remote }) => (local, remote),
_ => {
return Err(error::Error::ExecutionErr(
"bun bundle is missing the precomputed agent info".to_string(),
))
}
};
(local_path, remote_path)
}
};
let (cache, logs) =
windmill_common::worker::load_cache(&local_path, &remote_path, false).await;
@@ -920,7 +930,7 @@ pub async fn handle_bun_job(
canceled_by,
&job.id,
&job.workspace_id,
Some(db),
Some(conn),
job_dir,
worker_name,
common_bun_proc_envs.clone(),
@@ -932,13 +942,13 @@ pub async fn handle_bun_job(
} else {
// if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry {
let logs1 = "\n\n--- BUN INSTALL ---\n".to_string();
append_logs(&job.id, &job.workspace_id, logs1, db).await;
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
let _ = gen_bun_lockfile(
mem_peak,
canceled_by,
&job.id,
&job.workspace_id,
Some(db),
Some(conn),
&client.token,
job.runnable_path(),
job_dir,
@@ -1108,12 +1118,13 @@ try {{
let reserved_variables_args_out_f = async {
let args_and_out_f = async {
if !annotation.native {
create_args_and_out_file(&client, job, job_dir, db).await?;
create_args_and_out_file(&client, job, job_dir, conn).await?;
}
Ok(()) as Result<()>
};
let reserved_variables_f = async {
let vars = get_reserved_variables(job, &client.token, db, parent_runnable_path).await?;
let vars =
get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
Ok(vars) as Result<HashMap<String, String>>
};
let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?;
@@ -1176,7 +1187,7 @@ try {{
&job.workspace_id,
&job.id,
worker_name,
Some(db.clone()),
Some(conn),
job.timeout,
mem_peak,
canceled_by,
@@ -1225,7 +1236,7 @@ try {{
&job.workspace_id,
&job.id,
worker_name,
db,
conn,
job.timeout,
mem_peak,
canceled_by,
@@ -1255,7 +1266,7 @@ try {{
.join("\n"));
let js_code = read_file_content(&format!("{job_dir}/main.js")).await?;
let started_at = Instant::now();
let args = crate::common::build_args_map(job, client, db)
let args = crate::common::build_args_map(job, client, conn)
.await?
.map(sqlx::types::Json);
let job_args = if args.is_some() {
@@ -1264,7 +1275,7 @@ try {{
job.args.as_ref()
};
append_logs(&job.id, &job.workspace_id, format!("{init_logs}\n"), db).await;
append_logs(&job.id, &job.workspace_id, format!("{init_logs}\n"), conn).await;
let result = crate::js_eval::eval_fetch_timeout(
env_code,
@@ -1274,7 +1285,7 @@ try {{
job.script_entrypoint_override.clone(),
job.id,
job.timeout,
db,
conn,
mem_peak,
canceled_by,
worker_name,
@@ -1290,7 +1301,7 @@ try {{
return Ok(result);
}
}
append_logs(&job.id, &job.workspace_id, init_logs, db).await;
append_logs(&job.id, &job.workspace_id, init_logs, conn).await;
//do not cache local dependencies
let child = if !*DISABLE_NSJAIL {
@@ -1422,7 +1433,7 @@ try {{
handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
@@ -1532,7 +1543,7 @@ pub async fn start_worker(
annotation.nodejs = true;
let context = variables::get_reserved_variables(
db,
&Connection::from(db.clone()),
w_id,
&token,
"dedicated_worker@windmill.dev",
@@ -1583,7 +1594,7 @@ pub async fn start_worker(
&mut canceled_by,
&Uuid::nil(),
&w_id,
Some(db),
Some(&Connection::from(db.clone())),
job_dir,
worker_name,
common_bun_proc_envs.clone(),
@@ -1600,7 +1611,7 @@ pub async fn start_worker(
&mut canceled_by,
&Uuid::nil(),
&w_id,
Some(db),
Some(&Connection::from(db.clone())),
token,
&script_path,
job_dir,
@@ -1701,7 +1712,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) {
w_id,
&Uuid::nil(),
worker_name,
db,
&Connection::from(db.clone()),
None,
&mut mem_peak,
&mut canceled_by,
backend/windmill-worker/src/common.rs
+123 -67
View File
@@ -20,7 +20,8 @@ use windmill_common::s3_helpers::{
};
use windmill_common::variables::{build_crypt_with_key_suffix, decrypt};
use windmill_common::worker::{
to_raw_value, write_file, CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG,
to_raw_value, update_ping_for_failed_init_script_query, write_file, Connection, Ping, PingType,
CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG,
};
use windmill_common::{
cache::{Cache, RawData},
@@ -40,6 +41,7 @@ use windmill_common::{variables, DB};
use tokio::{io::AsyncWriteExt, process::Child, time::Instant};
use crate::agent_workers::UPDATE_PING_URL;
use crate::{
AuthedClient, DISABLE_NSJAIL, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE, MAX_TIMEOUT_DURATION,
PATH_ENV,
@@ -48,10 +50,10 @@ use crate::{
pub async fn build_args_map<'a>(
job: &'a MiniPulledJob,
client: &AuthedClient,
db: &Pool<Postgres>,
conn: &Connection,
) -> error::Result<Option<HashMap<String, Box<RawValue>>>> {
if let Some(args) = &job.args {
return transform_json(client, &job.workspace_id, &args.0, &job, db).await;
return transform_json(client, &job.workspace_id, &args.0, &job, conn).await;
}
return Ok(None);
}
@@ -76,10 +78,10 @@ pub fn check_executor_binary_exists(
pub async fn build_args_values(
job: &MiniPulledJob,
client: &AuthedClient,
db: &Pool<Postgres>,
conn: &Connection,
) -> error::Result<HashMap<String, serde_json::Value>> {
if let Some(args) = &job.args {
transform_json_as_values(client, &job.workspace_id, &args.0, job, db).await
transform_json_as_values(client, &job.workspace_id, &args.0, job, conn).await
} else {
Ok(HashMap::new())
}
@@ -90,10 +92,10 @@ pub async fn create_args_and_out_file(
client: &AuthedClient,
job: &MiniPulledJob,
job_dir: &str,
db: &Pool<Postgres>,
conn: &Connection,
) -> Result<(), Error> {
if let Some(args) = job.args.as_ref() {
if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, db).await? {
if let Some(x) = transform_json(client, &job.workspace_id, &args.0, job, conn).await? {
write_file(
job_dir,
"args.json",
@@ -131,7 +133,7 @@ pub async fn transform_json<'a>(
workspace: &str,
vs: &'a HashMap<String, Box<RawValue>>,
job: &MiniPulledJob,
db: &Pool<Postgres>,
db: &Connection,
) -> error::Result<Option<HashMap<String, Box<RawValue>>>> {
let mut has_match = false;
for (_, v) in vs {
@@ -168,7 +170,7 @@ pub async fn transform_json_as_values<'a>(
workspace: &str,
vs: &'a HashMap<String, Box<RawValue>>,
job: &MiniPulledJob,
db: &Pool<Postgres>,
db: &Connection,
) -> error::Result<HashMap<String, serde_json::Value>> {
let mut r: HashMap<String, serde_json::Value> = HashMap::new();
for (k, v) in vs {
@@ -236,7 +238,7 @@ pub async fn transform_json_value(
workspace: &str,
v: Value,
job: &MiniPulledJob,
db: &Pool<Postgres>,
conn: &Connection,
) -> error::Result<Value> {
match v {
Value::String(y) if y.starts_with("$var:") => {
@@ -267,20 +269,32 @@ pub async fn transform_json_value(
})
}
Value::String(y) if y.starts_with("$encrypted:") => {
let encrypted = y.strip_prefix("$encrypted:").unwrap();
match conn {
Connection::Sql(db) => {
let encrypted = y.strip_prefix("$encrypted:").unwrap();
let root_job_id =
get_root_job_id(&job.flow_innermost_root_job.unwrap_or_else(|| job.id), db).await?;
let mc = build_crypt_with_key_suffix(&db, &job.workspace_id, &root_job_id.to_string())
.await?;
decrypt(&mc, encrypted.to_string()).and_then(|x| {
serde_json::from_str(&x).map_err(|e| Error::internal_err(e.to_string()))
})
let root_job_id =
get_root_job_id(&job.flow_innermost_root_job.unwrap_or_else(|| job.id), db)
.await?;
let mc = build_crypt_with_key_suffix(
&db,
&job.workspace_id,
&root_job_id.to_string(),
)
.await?;
decrypt(&mc, encrypted.to_string()).and_then(|x| {
serde_json::from_str(&x).map_err(|e| Error::internal_err(e.to_string()))
})
}
Connection::Http(_) => {
Err(Error::NotFound("Http connection not supported".to_string()))
}
}
// let path = y.strip_prefix("$res:").unwrap();
}
Value::String(y) if y.starts_with("$") => {
let variables = get_reserved_variables(job, &client.token, &db, None).await?;
let variables = get_reserved_variables(job, &client.token, conn, None).await?;
let name = y.strip_prefix("$").unwrap();
@@ -295,7 +309,7 @@ pub async fn transform_json_value(
for (a, b) in m.clone().into_iter() {
m.insert(
a.clone(),
transform_json_value(&a, client, workspace, b, job, &db).await?,
transform_json_value(&a, client, workspace, b, job, conn).await?,
);
}
Ok(Value::Object(m))
@@ -387,16 +401,21 @@ pub fn capitalize(s: &str) -> String {
pub async fn get_reserved_variables(
job: &MiniPulledJob,
token: &str,
db: &sqlx::Pool<sqlx::Postgres>,
db: &Connection,
parent_runnable_path: Option<String>,
) -> Result<HashMap<String, String>, Error> {
let flow_path = if parent_runnable_path.is_some() {
parent_runnable_path
} else if let Some(uuid) = job.parent_job {
sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", uuid)
.fetch_optional(db)
.await?
.flatten()
match db {
Connection::Sql(db) => {
sqlx::query_scalar!("SELECT runnable_path FROM v2_job WHERE id = $1", uuid)
.fetch_optional(db)
.await?
.flatten()
}
Connection::Http(_) => None,
}
} else {
None
};
@@ -457,26 +476,57 @@ pub fn sizeof_val(v: &serde_json::Value) -> usize {
}
pub async fn update_worker_ping_for_failed_init_script(
db: &DB,
conn: &Connection,
worker_name: &str,
last_job_id: Uuid,
) {
if let Err(e) = sqlx::query!(
"UPDATE worker_ping SET
ping_at = now(),
jobs_executed = 1,
current_job_id = $1,
current_job_workspace_id = 'admins'
WHERE worker = $2",
last_job_id,
worker_name
)
.execute(db)
.await
{
tracing::error!("Error updating worker ping for failed init script: {e:?}");
match conn {
Connection::Sql(db) => {
if let Err(e) =
update_ping_for_failed_init_script_query(worker_name, last_job_id, db).await
{
tracing::error!("Error updating worker ping for failed init script: {e:?}");
}
}
Connection::Http(client) => {
if let Err(e) = client
.post::<_, ()>(
UPDATE_PING_URL,
&Ping {
last_job_executed: Some(last_job_id),
last_job_workspace_id: None,
worker_instance: None,
ip: None,
tags: None,
dw: None,
jobs_executed: None,
occupancy_rate: None,
occupancy_rate_15s: None,
occupancy_rate_5m: None,
occupancy_rate_30m: None,
version: None,
vcpus: None,
memory: None,
memory_usage: None,
wm_memory_usage: None,
ping_type: PingType::InitScript,
},
)
.await
{
tracing::error!("Error updating worker ping for failed init script: {e:?}");
}
}
}
}
pub fn error_to_value(err: Error) -> serde_json::Value {
match err {
Error::JsonErr(err) => err,
_ => json!({"message": err.to_string(), "name": "InternalErr"}),
}
}
pub struct OccupancyMetrics {
pub running_job_started_at: Option<Instant>,
pub total_duration_of_running_jobs: f32,
@@ -484,6 +534,13 @@ pub struct OccupancyMetrics {
pub start_time: Instant,
}
pub struct OccupancyResult {
pub occupancy_rate: f32,
pub occupancy_rate_15s: Option<f32>,
pub occupancy_rate_5m: Option<f32>,
pub occupancy_rate_30m: Option<f32>,
}
impl OccupancyMetrics {
pub fn new(start_time: Instant) -> Self {
OccupancyMetrics {
@@ -494,7 +551,7 @@ impl OccupancyMetrics {
}
}
pub fn update_occupancy_metrics(&mut self) -> (f32, Option<f32>, Option<f32>, Option<f32>) {
pub fn update_occupancy_metrics(&mut self) -> OccupancyResult {
let metrics = self;
let current_occupied_duration = metrics
.running_job_started_at
@@ -545,12 +602,12 @@ impl OccupancyMetrics {
.worker_occupancy_rate_history
.push((total_occupation, elapsed));
(
OccupancyResult {
occupancy_rate,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
)
}
}
}
@@ -561,15 +618,19 @@ pub async fn start_child_process(mut cmd: Command, executable: &str) -> Result<C
}
pub async fn resolve_job_timeout(
_db: &Pool<Postgres>,
_conn: &Connection,
_w_id: &str,
_job_id: Uuid,
custom_timeout_secs: Option<i32>,
) -> (Duration, Option<String>, bool) {
let mut warn_msg: Option<String> = None;
#[cfg(feature = "cloud")]
let cloud_premium_workspace =
*CLOUD_HOSTED && windmill_common::workspaces::is_premium_workspace(_db, _w_id).await;
let cloud_premium_workspace = *CLOUD_HOSTED
&& windmill_common::workspaces::is_premium_workspace(
_conn.as_sql().expect("cloud cannot use http connection"),
_w_id,
)
.await;
#[cfg(not(feature = "cloud"))]
let cloud_premium_workspace = false;
@@ -1013,7 +1074,7 @@ pub async fn par_install_language_dependencies<'a>(
job_id: &'a Uuid,
w_id: &'a str,
worker_name: &'a str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
) -> anyhow::Result<()> {
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
let _ = (platform_agnostic, language_name);
@@ -1038,7 +1099,7 @@ pub async fn par_install_language_dependencies<'a>(
counter_arc: Arc<tokio::sync::Mutex<usize>>,
total_to_install: usize,
instant: std::time::Instant,
db: Pool<Postgres>,
conn: &Connection,
) {
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
{
@@ -1074,7 +1135,7 @@ pub async fn par_install_language_dependencies<'a>(
if s3_push { " > (S3) " } else { "" },
instant.elapsed().as_millis(),
),
db,
conn,
)
.await;
// Drop lock, so next print success can fire
@@ -1128,18 +1189,13 @@ pub async fn par_install_language_dependencies<'a>(
job_id,
w_id,
format!("\n--- INSTALLATION ---\n\nTo be installed:\n\n"),
db.clone(),
conn,
)
.await;
to_be_installed_is_used = true;
}
windmill_queue::append_logs(
job_id,
w_id,
format!("- {display_name}\n"),
db.clone(),
)
.await;
windmill_queue::append_logs(job_id, w_id, format!("- {display_name}\n"), conn)
.await;
not_installed.push(NotInstalledDependency {
path,
custom_name,
@@ -1164,7 +1220,7 @@ pub async fn par_install_language_dependencies<'a>(
job_id,
w_id,
format!("\nLooking for packages on S3:\n"),
db.clone(),
conn,
)
.await;
}
@@ -1245,7 +1301,7 @@ pub async fn par_install_language_dependencies<'a>(
custom_name,
job_id_2,
w_id_2,
db_2,
conn_2,
counter_arc,
language_name,
installer_executable_name,
@@ -1257,7 +1313,7 @@ pub async fn par_install_language_dependencies<'a>(
custom_name.clone(),
job_id.clone(),
w_id.to_owned(),
db.clone(),
conn.clone(),
counter_arc.clone(),
language_name.to_owned(),
installer_executable_name.to_owned(),
@@ -1303,7 +1359,7 @@ pub async fn par_install_language_dependencies<'a>(
counter_arc,
total_to_install,
start,
db_2,
&conn_2,
)
.await;
return;
@@ -1321,7 +1377,7 @@ pub async fn par_install_language_dependencies<'a>(
};
if let Err(e) = crate::handle_child::handle_child(
&job_id_2,
&db_2,
&conn_2,
// TODO: Return mem_peak
&mut 0,
// TODO: Return canceld_by_ref
@@ -1342,7 +1398,7 @@ pub async fn par_install_language_dependencies<'a>(
&job_id_2,
&w_id_2,
format!("error while installing {}: {e:?}", &display_name_2),
db_2.clone(),
&conn_2,
)
.await;
} else {
@@ -1372,7 +1428,7 @@ pub async fn par_install_language_dependencies<'a>(
counter_arc,
total_to_install,
start,
db_2,
&conn_2,
)
.await;
// TODO: Refactor
@@ -1428,7 +1484,7 @@ pub async fn par_install_language_dependencies<'a>(
job_id,
w_id,
format!("\n\nFetching {} packages...\n", not_pulled_copy.len()),
db.clone(),
&conn,
)
.await;
let cmd = callback(not_pulled_copy.clone())?;
@@ -1439,7 +1495,7 @@ pub async fn par_install_language_dependencies<'a>(
if let Err(e) = crate::handle_child::handle_child(
// &job_id,
&Uuid::nil(),
&db,
&conn,
// TODO: Return mem_peak
&mut 0,
// TODO: Return canceld_by_ref
@@ -1516,7 +1572,7 @@ pub async fn par_install_language_dependencies<'a>(
"\nDone. Time spent on installation phase: {}ms\n",
total_time
),
db,
conn,
)
.await;
}
backend/windmill-worker/src/csharp_executor.rs
+16 -16
View File
@@ -67,7 +67,7 @@ pub async fn generate_nuget_lockfile(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job_dir: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut OccupancyMetrics,
@@ -115,7 +115,7 @@ pub async fn generate_nuget_lockfile(
let gen_lockfile_process = start_child_process(gen_lockfile_cmd, DOTNET_PATH.as_str()).await?;
handle_child(
job_id,
db,
conn,
mem_peak,
canceled_by,
gen_lockfile_process,
@@ -150,7 +150,7 @@ pub async fn generate_nuget_lockfile(
_mem_peak: &mut i32,
_canceled_by: &mut Option<CanceledBy>,
_job_dir: &str,
_db: &sqlx::Pool<sqlx::Postgres>,
_conn: &Connection,
_worker_name: &str,
_w_id: &str,
_occupancy_metrics: &mut OccupancyMetrics,
@@ -311,7 +311,7 @@ async fn build_cs_proj(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job_dir: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
worker_name: &str,
w_id: &str,
base_internal_url: &str,
@@ -371,7 +371,7 @@ async fn build_cs_proj(
let build_cs_process = start_child_process(build_cs_cmd, DOTNET_PATH.as_str()).await?;
handle_child(
job_id,
db,
conn,
mem_peak,
canceled_by,
build_cs_process,
@@ -385,7 +385,7 @@ async fn build_cs_proj(
None,
)
.await?;
append_logs(job_id, w_id, "\n\n", db).await;
append_logs(job_id, w_id, "\n\n", conn).await;
if let Err(e) = std::fs::remove_file(Path::new(job_dir).join("nuget.config")) {
if e.kind() != io::ErrorKind::NotFound {
Err(anyhow!("Error erasing nuget.config: {}", e))?;
@@ -428,6 +428,7 @@ fn remove_lines_from_text(contents: &str, indices_to_remove: Vec<usize>) -> Stri
result.join("\n")
}
use windmill_common::worker::Connection;
use windmill_queue::MiniPulledJob;
#[cfg(not(feature = "csharp"))]
@@ -435,7 +436,7 @@ pub async fn handle_csharp_job(
_mem_peak: &mut i32,
_canceled_by: &mut Option<CanceledBy>,
_job: &MiniPulledJob,
_db: &sqlx::Pool<sqlx::Postgres>,
_conn: &Connection,
_client: &AuthedClient,
_parent_runnable_path: Option<String>,
_inner_content: &str,
@@ -449,13 +450,12 @@ pub async fn handle_csharp_job(
) -> Result<Box<RawValue>, Error> {
Err(anyhow!("C# is not available because the feature is not enabled").into())
}
#[cfg(feature = "csharp")]
pub async fn handle_csharp_job(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job: &MiniPulledJob,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
client: &AuthedClient,
parent_runnable_path: Option<String>,
inner_content: &str,
@@ -495,7 +495,7 @@ pub async fn handle_csharp_job(
cache_logs
} else {
let logs1 = format!("{cache_logs}\n\n--- DOTNET BUILD ---\n");
append_logs(&job.id, &job.workspace_id, logs1, db).await;
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
let (reqs, lines_to_remove) = parse_csharp_reqs(inner_content);
for req in &reqs {
@@ -507,7 +507,7 @@ pub async fn handle_csharp_job(
req.0,
req.1.as_ref().unwrap_or(&"".to_string())
),
db,
conn,
)
.await;
}
@@ -525,7 +525,7 @@ pub async fn handle_csharp_job(
mem_peak,
canceled_by,
job_dir,
db,
conn,
worker_name,
&job.workspace_id,
base_internal_url,
@@ -535,13 +535,13 @@ pub async fn handle_csharp_job(
.await?
};
create_args_and_out_file(client, job, job_dir, db).await?;
create_args_and_out_file(client, job, job_dir, conn).await?;
let logs2 = format!("{cache_logs}\n\n--- C# CODE EXECUTION ---\n");
append_logs(&job.id, &job.workspace_id, format!("{}\n", logs2), db).await;
append_logs(&job.id, &job.workspace_id, format!("{}\n", logs2), conn).await;
let reserved_variables =
get_reserved_variables(job, &client.token, db, parent_runnable_path).await?;
get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
let child = if !*DISABLE_NSJAIL {
write_file(
@@ -627,7 +627,7 @@ pub async fn handle_csharp_job(
handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
backend/windmill-worker/src/dedicated_worker.rs
+25 -25
View File
@@ -29,9 +29,7 @@ use windmill_queue::MiniPulledJob;
use anyhow::Context;
use crate::{
common::start_child_process, JobCompleted, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS,
};
use crate::{common::start_child_process, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS};
use futures::{future, Future};
use std::{collections::HashMap, task::Poll};
@@ -78,7 +76,7 @@ pub async fn handle_dedicated_process(
) -> std::result::Result<(), error::Error> {
//do not cache local dependencies
use windmill_queue::MiniPulledJob;
use windmill_queue::{JobCompleted, MiniPulledJob};
use crate::{handle_child::process_status, PROXY_ENVS};
let cmd_name = format!("dedicated {command_path}");
@@ -187,16 +185,16 @@ pub async fn handle_dedicated_process(
match serde_json::from_str::<Box<serde_json::value::RawValue>>(&line.replace("wm_res[success]:", "").replace("wm_res[error]:", "")) {
Ok(result) => {
let result = Arc::new(result);
append_logs(&job.id, &job.workspace_id, logs.clone(), db).await;
append_logs(&job.id, &job.workspace_id, logs.clone(), &db.into()).await;
if line.starts_with("wm_res[success]:") {
job_completed_tx.send(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap()
job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: true, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap()
} else {
job_completed_tx.send(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap()
job_completed_tx.send_job(JobCompleted { job , result, result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap()
}
},
Err(e) => {
tracing::error!("Could not deserialize job result `{line}`: {e:?}");
job_completed_tx.send(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap();
job_completed_tx.send_job(JobCompleted { job , result: Arc::new(to_raw_value(&serde_json::json!({"error": format!("Could not deserialize job result `{line}`: {e:?}")}))), result_columns: None, mem_peak: 0, canceled_by: None, success: false, cached_res_path: None, token: token.to_string(), duration: None }).await.unwrap();
},
};
logs = init_log.clone();
@@ -397,13 +395,16 @@ async fn spawn_dedicated_workers_for_flow(
}
}
FlowModuleValue::FlowScript { id, language, .. } => {
let spawn = cache::flow::fetch_script(db, *id).await.map(|data| {
SpawnWorker::RawScript {
path: "".to_string(),
content: data.code.clone(),
lock: data.lock.clone(),
lang: *language,
}
let spawn = cache::flow::fetch_script(
&windmill_common::worker::Connection::Sql(db.clone()),
*id,
)
.await
.map(|data| SpawnWorker::RawScript {
path: "".to_string(),
content: data.code.clone(),
lock: data.lock.clone(),
lang: *language,
});
match spawn {
Ok(spawn) => {
@@ -571,7 +572,7 @@ async fn spawn_dedicated_worker(
};
use windmill_queue::MiniPulledJob;
use crate::{build_envs, get_script_content_by_hash, ContentReqLangEnvs, JOB_TOKEN};
use crate::{build_envs, get_script_content_by_hash, ContentReqLangEnvs};
#[cfg(not(feature = "enterprise"))]
{
@@ -586,7 +587,7 @@ async fn spawn_dedicated_worker(
std::sync::Arc<MiniPulledJob>,
>(MAX_BUFFERED_DEDICATED_JOBS);
let killpill_rx = killpill_rx.resubscribe();
let db = db.clone();
let db2 = db.clone();
let base_internal_url = base_internal_url.to_string();
let worker_name = worker_name.to_string();
let job_completed_tx = job_completed_tx.clone();
@@ -613,11 +614,11 @@ async fn spawn_dedicated_worker(
let (content, lock, language, envs, codebase) = match sw.clone() {
SpawnWorker::Script { path, hash } => {
let q = if let Some(hash) = hash {
get_script_content_by_hash(&hash, &w_id, &db).await.map(
|r: ContentReqLangEnvs| {
get_script_content_by_hash(&hash, &w_id, &db2.into())
.await
.map(|r: ContentReqLangEnvs| {
Some((r.content, r.lockfile, r.language, r.envs, r.codebase))
},
)
})
} else {
sqlx::query_as::<_, (String, Option<String>, Option<ScriptLang>, Option<Vec<String>>, bool, Option<ScriptHash>)>(
"SELECT content, lock, language, envs, codebase IS NOT NULL, hash FROM script WHERE path = $1 AND workspace_id = $2 AND
@@ -626,7 +627,7 @@ async fn spawn_dedicated_worker(
)
.bind(&path)
.bind(&w_id)
.fetch_optional(&db)
.fetch_optional(&db2)
.await
.map_err(|e| Error::internal_err(format!("expected content and lock: {e:#}")))
.map(|x| x.map(|y| (y.0, y.1, y.2, y.3, if y.4 { y.5.map(|z| z.to_string()) } else { None })))
@@ -658,10 +659,9 @@ async fn spawn_dedicated_worker(
_ => return None,
}
let db = db.clone();
let handle = tokio::spawn(async move {
let token = if let Some(token) = JOB_TOKEN.as_ref() {
token.clone()
} else {
let token = {
let token = rd_string(32);
if let Err(e) = sqlx::query_scalar!(
"INSERT INTO token
backend/windmill-worker/src/deno_executor.rs
+12 -8
View File
@@ -15,8 +15,11 @@ use crate::{
PATH_ENV, TZ_ENV,
};
use tokio::{fs::File, io::AsyncReadExt, process::Command};
use windmill_common::error::{self};
use windmill_common::{error::Result, worker::write_file, BASE_URL};
use windmill_common::{
error::{self},
worker::Connection,
};
use windmill_parser::Typ;
lazy_static::lazy_static! {
@@ -97,7 +100,7 @@ pub async fn generate_deno_lock(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job_dir: &str,
db: Option<&sqlx::Pool<sqlx::Postgres>>,
db: Option<&Connection>,
w_id: &str,
worker_name: &str,
base_internal_url: &str,
@@ -179,7 +182,7 @@ pub async fn handle_deno_job(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job: &MiniPulledJob,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
client: &AuthedClient,
parent_runnable_path: Option<String>,
job_dir: &str,
@@ -192,7 +195,7 @@ pub async fn handle_deno_job(
) -> error::Result<Box<RawValue>> {
// let mut start = Instant::now();
let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string();
append_logs(&job.id, &job.workspace_id, logs1, db).await;
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
let main_override = job.script_entrypoint_override.as_deref();
let apply_preprocessor = !job.is_flow_step() && job.preprocessed == Some(false);
@@ -316,11 +319,12 @@ try {{
let reserved_variables_args_out_f = async {
let args_and_out_f = async {
create_args_and_out_file(&client, job, job_dir, db).await?;
create_args_and_out_file(&client, job, job_dir, conn).await?;
Ok(()) as Result<()>
};
let reserved_variables_f = async {
let vars = get_reserved_variables(job, &client.token, db, parent_runnable_path).await?;
let vars =
get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
Ok(vars) as Result<HashMap<String, String>>
};
let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?;
@@ -404,7 +408,7 @@ try {{
// start = Instant::now();
handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
@@ -514,7 +518,7 @@ pub async fn start_worker(
let common_deno_proc_envs = get_common_deno_proc_envs(&token, base_internal_url).await;
let context = variables::get_reserved_variables(
db,
&db.into(),
w_id,
&token,
"dedicated_worker@windmill.dev",
backend/windmill-worker/src/go_executor.rs
+38 -32
View File
@@ -8,7 +8,7 @@ use uuid::Uuid;
use windmill_common::{
error::{self, Error},
utils::calculate_hash,
worker::{save_cache, write_file},
worker::{save_cache, write_file, Connection},
};
use windmill_parser_go::{parse_go_imports, REQUIRE_PARSE};
use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
@@ -36,7 +36,7 @@ pub async fn handle_go_job(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job: &MiniPulledJob,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
client: &AuthedClient,
parent_runnable_path: Option<String>,
inner_content: &str,
@@ -78,7 +78,7 @@ pub async fn handle_go_job(
let cache_logs = if !cache {
let logs1 = format!("{cache_logs}\n\n--- GO DEPENDENCIES SETUP ---\n");
append_logs(&job.id, &job.workspace_id, logs1, db).await;
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
install_go_dependencies(
&job.id,
@@ -86,7 +86,7 @@ pub async fn handle_go_job(
mem_peak,
canceled_by,
job_dir,
db,
conn,
true,
skip_go_mod,
skip_tidy,
@@ -96,7 +96,7 @@ pub async fn handle_go_job(
)
.await?;
create_args_and_out_file(client, job, job_dir, db).await?;
create_args_and_out_file(client, job, job_dir, conn).await?;
{
let sig = windmill_parser_go::parse_go_sig(&inner_content)?;
@@ -202,7 +202,7 @@ func Run(req Req) (interface{{}}, error){{
let build_go_process = start_child_process(build_go_cmd, GO_PATH.as_str()).await?;
handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
build_go_process,
@@ -245,15 +245,15 @@ func Run(req Req) (interface{{}}, error){{
))
})?;
create_args_and_out_file(client, job, job_dir, db).await?;
create_args_and_out_file(client, job, job_dir, conn).await?;
cache_logs
};
let logs2 = format!("{cache_logs}\n\n--- GO CODE EXECUTION ---\n");
append_logs(&job.id, &job.workspace_id, logs2, db).await;
append_logs(&job.id, &job.workspace_id, logs2, conn).await;
let reserved_variables =
get_reserved_variables(job, &client.token, db, parent_runnable_path).await?;
get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
let child = if !*DISABLE_NSJAIL {
let _ = write_file(
@@ -306,7 +306,7 @@ func Run(req Req) (interface{{}}, error){{
};
handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
@@ -351,7 +351,7 @@ pub async fn install_go_dependencies(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job_dir: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
non_dep_job: bool,
skip_go_mod: bool,
has_sum: bool,
@@ -371,7 +371,7 @@ pub async fn install_go_dependencies(
handle_child(
job_id,
db,
conn,
mem_peak,
canceled_by,
child_process,
@@ -409,20 +409,22 @@ pub async fn install_go_dependencies(
let mut skip_tidy = has_sum;
if !has_sum {
if let Some(cached) = sqlx::query_scalar!(
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
hash
)
.fetch_optional(db)
.await?
{
let logs1 = format!("\nfound cached resolution: {}", hash);
append_logs(&job_id, w_id, logs1, db).await;
gen_go_mod(code, job_dir, &cached).await?;
skip_tidy = true;
new_lockfile = false;
} else {
new_lockfile = true;
if let Some(db) = conn.as_sql() {
if let Some(cached) = sqlx::query_scalar!(
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
hash
)
.fetch_optional(db)
.await?
{
let logs1 = format!("\nfound cached resolution: {}", hash);
append_logs(&job_id, w_id, logs1, conn).await;
gen_go_mod(code, job_dir, &cached).await?;
skip_tidy = true;
new_lockfile = false;
} else {
new_lockfile = true;
}
}
}
@@ -438,7 +440,7 @@ pub async fn install_go_dependencies(
handle_child(
job_id,
db,
conn,
mem_peak,
canceled_by,
child_process,
@@ -469,11 +471,15 @@ pub async fn install_go_dependencies(
}
if non_dep_job {
sqlx::query!(
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
hash,
req_content
).fetch_optional(db).await?;
if let Some(db) = conn.as_sql() {
sqlx::query!(
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
hash,
req_content
)
.fetch_optional(db)
.await?;
}
return Ok(String::new());
} else {
backend/windmill-worker/src/graphql_executor.rs
+5 -5
View File
@@ -4,7 +4,7 @@ use anyhow::anyhow;
use futures::{stream, TryStreamExt};
use serde_json::{json, value::RawValue};
use sqlx::types::Json;
use windmill_common::worker::to_raw_value;
use windmill_common::worker::{to_raw_value, Connection};
use windmill_common::{error::Error, worker::CLOUD_HOSTED};
use windmill_parser_graphql::parse_graphql_sig;
use windmill_queue::{CanceledBy, MiniPulledJob};
@@ -37,13 +37,13 @@ pub async fn do_graphql(
job: &MiniPulledJob,
client: &AuthedClient,
query: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
occupation_metrics: &mut OccupancyMetrics,
) -> windmill_common::error::Result<Box<RawValue>> {
let args = build_args_map(job, client, db).await?.map(Json);
let args = build_args_map(job, client, conn).await?.map(Json);
let job_args = if args.is_some() {
args.as_ref()
} else {
@@ -81,7 +81,7 @@ pub async fn do_graphql(
}
}
let (timeout_duration, _, _) =
resolve_job_timeout(&db, &job.workspace_id, job.id, job.timeout).await;
resolve_job_timeout(&conn, &job.workspace_id, job.id, job.timeout).await;
let http_client = build_http_client(timeout_duration)?;
@@ -150,7 +150,7 @@ pub async fn do_graphql(
let r = run_future_with_polling_update_job_poller(
job.id,
job.timeout,
db,
conn,
mem_peak,
canceled_by,
result_f,
backend/windmill-worker/src/handle_child.rs
+85 -86
View File
@@ -4,8 +4,9 @@ use futures::Future;
use nix::sys::signal::{self, Signal};
#[cfg(any(target_os = "linux", target_os = "macos"))]
use nix::unistd::Pid;
use windmill_common::agent_workers::PingJobStatusResponse;
use windmill_common::jobs::LARGE_LOG_THRESHOLD_SIZE;
use sqlx::{Pool, Postgres};
#[cfg(windows)]
use std::process::Stdio;
use tokio::fs::File;
@@ -15,7 +16,10 @@ use windmill_common::error::to_anyhow;
use windmill_common::error::{self, Error};
use windmill_common::worker::{get_windmill_memory_usage, get_worker_memory_usage, CLOUD_HOSTED};
use windmill_common::worker::{
get_windmill_memory_usage, get_worker_memory_usage, set_job_cancelled_query, Connection,
JobCancelled, CLOUD_HOSTED,
};
use windmill_queue::{append_logs, CanceledBy};
@@ -29,7 +33,6 @@ use std::{io, panic, time::Duration};
use tracing::{trace_span, Instrument};
use uuid::Uuid;
use windmill_common::DB;
#[cfg(feature = "enterprise")]
use windmill_common::job_metrics;
@@ -49,8 +52,9 @@ use futures::{
};
use crate::common::{resolve_job_timeout, OccupancyMetrics};
use crate::job_logger::{append_job_logs, append_with_limit, LARGE_LOG_THRESHOLD_SIZE};
use crate::job_logger::{append_job_logs, append_with_limit};
use crate::job_logger_ee::process_streaming_log_lines;
use crate::worker_utils::{ping_job_status, update_worker_ping_from_job};
use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM};
lazy_static::lazy_static! {
@@ -92,7 +96,7 @@ async fn kill_process_tree(pid: Option<u32>) -> Result<(), String> {
#[tracing::instrument(name="run_subprocess", level = "info", skip_all, fields(otel.name = %child_name))]
pub async fn handle_child(
job_id: &Uuid,
db: &Pool<Postgres>,
conn: &Connection,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
mut child: Child,
@@ -138,7 +142,7 @@ pub async fn handle_child(
* waiting for the child to exit normally */
let update_job = update_job_poller(
job_id,
db,
conn,
mem_peak,
canceled_by_ref,
Box::pin(stream::unfold((), move |_| async move {
@@ -184,15 +188,13 @@ pub async fn handle_child(
}
let (timeout_duration, timeout_warn_msg, is_job_specific) =
resolve_job_timeout(&db, w_id, job_id, custom_timeout).await;
resolve_job_timeout(&conn, w_id, job_id, custom_timeout).await;
if let Some(msg) = timeout_warn_msg {
append_logs(&job_id, w_id, msg.as_str(), db).await;
append_logs(&job_id, w_id, msg.as_str(), conn).await;
}
/* a future that completes when the child process exits */
let wait_on_child = async {
let db = db.clone();
let kill_reason = tokio::select! {
biased;
result = child.wait() => return result.map(Ok),
@@ -208,18 +210,33 @@ pub async fn handle_child(
let set_reason = async {
if matches!(kill_reason, KillReason::Timeout { .. }) {
if let Err(err) = sqlx::query!(
"UPDATE v2_job_queue
SET canceled_by = 'timeout'
, canceled_reason = $1
WHERE id = $2",
format!("duration > {}", timeout_duration.as_secs()),
job_id
)
.execute(&db)
.await
{
tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}");
match conn {
Connection::Sql(db) => {
if let Err(err) = set_job_cancelled_query(
job_id,
db,
"timeout",
&format!("duration > {}", timeout_duration.as_secs()),
)
.await
{
tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}");
}
}
Connection::Http(client) => {
if let Err(err) = client
.post::<_, ()>(
&format!("/api/agent_workers/set_job_cancelled/{}", job_id),
&JobCancelled {
canceled_by: "timeout".to_string(),
reason: format!("duration > {}", timeout_duration.as_secs()),
},
)
.await
{
tracing::error!(%job_id, %err, "error setting cancelation reason for job using http {job_id}: {err}");
}
}
}
}
};
@@ -389,11 +406,13 @@ pub async fn handle_child(
let worker_name = worker.to_string();
let w_id2 = w_id.to_string();
if let Some(buf) = &mut pipe_stdout {
buf.push_str(&joined);
(do_write, write_result) = tokio::spawn(async { }).remote_handle();
} else {
(do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
(do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, conn.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
}
if let Err(err) = result {
@@ -497,7 +516,7 @@ pub(crate) async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
pub async fn run_future_with_polling_update_job_poller<Fut, T, S>(
job_id: Uuid,
timeout: Option<i32>,
db: &DB,
conn: &Connection,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
result_f: Fut,
@@ -514,7 +533,7 @@ where
let update_job = update_job_poller(
job_id,
db,
conn,
mem_peak,
canceled_by_ref,
get_mem,
@@ -525,7 +544,7 @@ where
);
let timeout_ms = u64::try_from(
resolve_job_timeout(&db, &w_id, job_id, timeout)
resolve_job_timeout(&conn, &w_id, job_id, timeout)
.await
.0
.as_millis(),
@@ -560,7 +579,7 @@ pub enum UpdateJobPollingExit {
pub async fn update_job_poller<S>(
job_id: Uuid,
db: &DB,
conn: &Connection,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
mut get_mem: S,
@@ -574,8 +593,7 @@ where
{
let update_job_interval = Duration::from_millis(500);
let db = db.clone();
let conn = conn.clone();
let mut interval = interval(update_job_interval);
interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
@@ -597,22 +615,9 @@ where
tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024);
let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics());
if job_id != Uuid::nil() {
sqlx::query!(
"UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,
occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
&job_id,
&w_id,
memory_usage,
wm_memory_usage,
&worker_name,
occupancy.map(|x| x.0),
occupancy.and_then(|x| x.1),
occupancy.and_then(|x| x.2),
occupancy.and_then(|x| x.3),
)
.execute(&db)
.await
.expect("update worker ping");
if let Err(err) = update_worker_ping_from_job(&conn, &job_id, w_id, worker_name, memory_usage, wm_memory_usage, occupancy).await {
tracing::error!("Unable to update worker ping for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
}
}
}
let current_mem = get_mem.next().await.unwrap_or(0);
@@ -627,55 +632,49 @@ where
#[cfg(feature = "enterprise")]
{
if job_id != Uuid::nil() {
// tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
if i == 2 {
memory_metric_id = job_metrics::register_metric_for_job(
&db,
w_id.to_string(),
job_id,
"memory_kb".to_string(),
job_metrics::MetricKind::TimeseriesInt,
Some("Job Memory Footprint (kB)".to_string()),
)
.await;
}
if let Ok(ref metric_id) = memory_metric_id {
if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
if let Connection::Sql(ref db) = conn {
// tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
if i == 2 {
memory_metric_id = job_metrics::register_metric_for_job(
&db,
w_id.to_string(),
job_id,
"memory_kb".to_string(),
job_metrics::MetricKind::TimeseriesInt,
Some("Job Memory Footprint (kB)".to_string()),
)
.await;
}
if let Ok(ref metric_id) = memory_metric_id {
if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
}
}
}
}
}
if job_id != Uuid::nil() {
let (canceled_by, canceled_reason, already_completed) = sqlx::query!(
"UPDATE v2_job_runtime r SET
memory_peak = $1,
ping = now()
FROM v2_job_queue q
WHERE r.id = $2 AND q.id = r.id
RETURNING canceled_by, canceled_reason",
*mem_peak,
job_id
)
.map(|x| (x.canceled_by, x.canceled_reason, false))
.fetch_optional(&db)
.await
.unwrap_or_else(|e| {
tracing::error!(%e, "error updating job {job_id}: {e:#}");
Some((None, None, false))
})
.unwrap_or_else(|| {
// if the job is not in queue, it can only be in the completed_job so it is already complete
(None, None, true)
});
if already_completed {
if matches!(conn, Connection::Http(_)) {
if i % 4 != 0 {
// only ping every 4th time (2s) on http agent mode
continue;
}
}
let ping_job_status = ping_job_status(&conn, &job_id, Some(*mem_peak), if current_mem > 0 { Some(current_mem) } else { None }).await.unwrap_or_else(|e| {
tracing::error!("Unable to ping job status for job {job_id}. Error was: {:?}", e);
PingJobStatusResponse {
canceled_by: None,
canceled_reason: None,
already_completed: false,
}
});
if ping_job_status.already_completed {
return UpdateJobPollingExit::AlreadyCompleted
}
if canceled_by.is_some() {
if ping_job_status.canceled_by.is_some() {
canceled_by_ref.replace(CanceledBy {
username: canceled_by.clone(),
reason: canceled_reason.clone(),
username: ping_job_status.canceled_by.clone(),
reason: ping_job_status.canceled_reason.clone(),
});
break
}
backend/windmill-worker/src/java_executor.rs
+36 -30
View File
@@ -13,7 +13,7 @@ use uuid::Uuid;
use windmill_common::{
error::{self, Error},
utils::calculate_hash,
worker::{copy_dir_recursively, save_cache, write_file},
worker::{copy_dir_recursively, save_cache, write_file, Connection},
};
use windmill_parser::Arg;
use windmill_parser_java::parse_java_sig_meta;
@@ -44,7 +44,7 @@ pub(crate) struct JobHandlerInput<'a> {
pub canceled_by: &'a mut Option<CanceledBy>,
pub client: &'a AuthedClient,
pub parent_runnable_path: Option<String>,
pub db: &'a sqlx::Pool<sqlx::Postgres>,
pub conn: &'a Connection,
pub envs: HashMap<String, String>,
pub inner_content: &'a str,
pub job: &'a MiniPulledJob,
@@ -67,7 +67,7 @@ pub async fn handle_java_job<'a>(mut args: JobHandlerInput<'a>) -> Result<Box<Ra
&args.job.id,
&args.inner_content,
&args.job_dir,
&args.db,
&args.conn,
&args.job.workspace_id,
)
.await?;
@@ -91,11 +91,11 @@ pub async fn handle_java_job<'a>(mut args: JobHandlerInput<'a>) -> Result<Box<Ra
}
async fn prepare<'a>(
JobHandlerInput { job, db, job_dir, client, inner_content, .. }: &mut JobHandlerInput<'a>,
JobHandlerInput { job, conn, job_dir, client, inner_content, .. }: &mut JobHandlerInput<'a>,
) -> Result<(), Error> {
// Create needed files
{
create_args_and_out_file(&client, job, job_dir, db).await?;
create_args_and_out_file(&client, job, job_dir, conn).await?;
let app_path = format!("{}/src/main/java/net/script/", job_dir);
create_dir_all(&app_path).await?;
File::create(format!("{app_path}/App.java"))
@@ -120,7 +120,7 @@ pub async fn resolve<'a>(
job_id: &Uuid,
code: &str,
job_dir: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
w_id: &str,
) -> Result<String, Error> {
let deps = {
@@ -152,21 +152,23 @@ pub async fn resolve<'a>(
};
let req_hash = format!("java-{}", calculate_hash(&deps));
if let Some(cached) = sqlx::query_scalar!(
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
req_hash
)
.fetch_optional(db)
.await?
{
return Ok(cached);
if let Connection::Sql(db) = conn {
if let Some(cached) = sqlx::query_scalar!(
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
req_hash
)
.fetch_optional(db)
.await?
{
return Ok(cached);
}
}
let lock = {
append_logs(
job_id,
w_id,
format!("\n--- RESOLVING LOCKFILE ---\n"),
db.clone(),
&conn,
)
.await;
@@ -239,18 +241,22 @@ pub async fn resolve<'a>(
}
};
sqlx::query!(
if let Connection::Sql(db) = conn {
sqlx::query!(
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
req_hash,
lock.clone(),
).fetch_optional(db).await?;
lock.clone(),
)
.fetch_optional(db)
.await?;
}
append_logs(job_id, w_id, format!("\n{}", &lock), db.clone()).await;
append_logs(job_id, w_id, format!("\n{}", &lock), &conn).await;
Ok(lock)
}
async fn install<'a>(
JobHandlerInput { worker_name, job, db, job_dir, .. }: &mut JobHandlerInput<'a>,
JobHandlerInput { worker_name, job, conn, job_dir, .. }: &mut JobHandlerInput<'a>,
deps: String,
) -> Result<String, Error> {
let deps = deps
@@ -411,7 +417,7 @@ async fn install<'a>(
&job.id,
&job.workspace_id,
worker_name,
db,
conn,
)
.await?;
Ok(classpath)
@@ -424,7 +430,7 @@ async fn compile<'a>(
canceled_by,
worker_name,
job,
db,
conn,
job_dir,
client,
envs,
@@ -448,7 +454,7 @@ async fn compile<'a>(
))
}
let reserved_variables =
get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?;
get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?;
let hash = compute_hash(inner_content, *requirements_o);
let bin_path = format!("{}/{hash}", JAVA_CACHE_DIR);
let remote_path = format!("java_jar/{hash}");
@@ -474,7 +480,7 @@ async fn compile<'a>(
&job.id,
&job.workspace_id,
format!("\n--- COMPILING .JAVA FILES\n"),
db.clone(),
&conn,
)
.await;
@@ -514,7 +520,7 @@ async fn compile<'a>(
};
handle_child::handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
@@ -559,7 +565,7 @@ async fn run<'a>(
canceled_by,
worker_name,
job,
db,
conn,
job_dir,
shared_mount,
client,
@@ -571,14 +577,14 @@ async fn run<'a>(
classpath: &'a str,
) -> Result<(), Error> {
let reserved_variables =
get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?;
get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?;
let child = if !cfg!(windows) && !*DISABLE_NSJAIL {
append_logs(
&job.id,
&job.workspace_id,
format!("\n--- ISOLATED JAVA CODE EXECUTION ---\n"),
db.clone(),
&conn,
)
.await;
@@ -640,7 +646,7 @@ async fn run<'a>(
&job.id,
&job.workspace_id,
format!("\n--- JAVA CODE EXECUTION ---\n"),
db.clone(),
&conn,
)
.await;
@@ -697,7 +703,7 @@ async fn run<'a>(
};
handle_child::handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
backend/windmill-worker/src/job_logger.rs
+24 -23
View File
@@ -1,6 +1,7 @@
use regex::Regex;
use windmill_common::worker::CLOUD_HOSTED;
pub use windmill_common::jobs::LARGE_LOG_THRESHOLD_SIZE;
use windmill_common::worker::{Connection, CLOUD_HOSTED};
use windmill_queue::append_logs;
@@ -8,7 +9,6 @@ use std::sync::atomic::AtomicU32;
use std::sync::Arc;
use uuid::Uuid;
use windmill_common::DB;
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
use crate::job_logger_ee::default_disk_log_storage;
@@ -25,39 +25,40 @@ pub enum CompactLogs {
S3,
}
pub(crate) async fn append_job_logs(
pub async fn append_job_logs(
job_id: Uuid,
w_id: String,
logs: String,
db: DB,
conn: Connection,
must_compact_logs: bool,
total_size: Arc<AtomicU32>,
worker_name: String,
) -> () {
if must_compact_logs {
#[cfg(all(feature = "enterprise", feature = "parquet"))]
s3_storage(job_id, &w_id, &db, logs, total_size, &worker_name).await;
match conn {
Connection::Sql(db) if must_compact_logs => {
#[cfg(all(feature = "enterprise", feature = "parquet"))]
s3_storage(job_id, &w_id, &db, logs, total_size, &worker_name).await;
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
{
default_disk_log_storage(
job_id,
&w_id,
&db,
logs,
total_size,
CompactLogs::NotEE,
&worker_name,
)
.await;
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
{
default_disk_log_storage(
job_id,
&w_id,
&db,
logs,
total_size,
CompactLogs::NotEE,
&worker_name,
)
.await;
}
}
_ => {
append_logs(&job_id, w_id, logs, &conn).await;
}
} else {
append_logs(&job_id, w_id, logs, db).await;
}
}
pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000;
lazy_static::lazy_static! {
static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap();
pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true");
backend/windmill-worker/src/js_eval.rs
+9 -6
View File
@@ -45,7 +45,7 @@ use windmill_common::error::Error;
#[cfg(feature = "deno_core")]
use windmill_common::worker::{write_file, TMP_DIR};
use windmill_common::{flow_status::JobResult, DB};
use windmill_common::flow_status::JobResult;
use windmill_queue::CanceledBy;
use crate::{common::OccupancyMetrics, AuthedClient};
@@ -749,6 +749,9 @@ fn capture_proxy(s: &str) -> Option<(String, Option<(String, String)>)> {
)
})
}
use windmill_common::worker::Connection;
#[cfg(not(feature = "deno_core"))]
pub async fn eval_fetch_timeout(
_env_code: String,
@@ -758,7 +761,7 @@ pub async fn eval_fetch_timeout(
_script_entrypoint_override: Option<String>,
_job_id: Uuid,
_job_timeout: Option<i32>,
_db: &DB,
_conn: &Connection,
_mem_peak: &mut i32,
_canceled_by: &mut Option<CanceledBy>,
_worker_name: &str,
@@ -779,7 +782,7 @@ pub async fn eval_fetch_timeout(
script_entrypoint_override: Option<String>,
job_id: Uuid,
job_timeout: Option<i32>,
db: &DB,
conn: &Connection,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
@@ -825,7 +828,7 @@ pub async fn eval_fetch_timeout(
));
}
let db_ = db.clone();
let conn_ = conn.clone();
let w_id_ = w_id.to_string();
let result_f = tokio::task::spawn_blocking(move || {
let ops = vec![op_get_static_args(), op_log()];
@@ -921,7 +924,7 @@ pub async fn eval_fetch_timeout(
"{extra_logs}{}",
js_runtime.op_state().borrow().borrow::<LogString>().s
),
db_,
&conn_,
)
.await;
@@ -936,7 +939,7 @@ pub async fn eval_fetch_timeout(
let res = run_future_with_polling_update_job_poller(
job_id,
job_timeout,
db,
conn,
mem_peak,
canceled_by,
async { result_f.await? },
backend/windmill-worker/src/lib.rs
+8 -6
View File
@@ -5,13 +5,11 @@ mod mssql_executor;
#[cfg(feature = "enterprise")]
mod snowflake_executor;
mod agent_workers;
#[cfg(feature = "python")]
mod ansible_executor;
mod bash_executor;
#[cfg(feature = "nu")]
mod nu_executor;
#[cfg(feature = "java")]
mod java_executor;
@@ -28,11 +26,13 @@ mod global_cache;
mod go_executor;
mod graphql_executor;
mod handle_child;
mod job_logger;
pub mod job_logger;
mod job_logger_ee;
mod js_eval;
#[cfg(feature = "mysql")]
mod mysql_executor;
#[cfg(feature = "nu")]
mod nu_executor;
#[cfg(feature = "oracledb")]
mod oracledb_executor;
mod otel_ee;
@@ -41,7 +41,7 @@ mod pg_executor;
mod php_executor;
#[cfg(feature = "python")]
mod python_executor;
mod result_processor;
pub mod result_processor;
#[cfg(feature = "rust")]
mod rust_executor;
mod sanitized_sql_params;
@@ -49,12 +49,14 @@ mod schema;
mod worker;
mod worker_flow;
mod worker_lockfiles;
mod worker_utils;
pub use worker::*;
pub use result_processor::handle_job_error;
pub use bun_executor::{
get_common_bun_proc_envs, install_bun_lockfile, prebundle_bun_script, prepare_job_dir,
compute_bundle_local_and_remote_path, get_common_bun_proc_envs, install_bun_lockfile,
prebundle_bun_script, prepare_job_dir,
};
pub use deno_executor::generate_deno_lock;
backend/windmill-worker/src/mssql_executor.rs
+5 -5
View File
@@ -10,7 +10,7 @@ use tokio_util::compat::TokioAsyncWriteCompatExt;
use uuid::Uuid;
use windmill_common::error::to_anyhow;
use windmill_common::error::{self, Error};
use windmill_common::worker::to_raw_value;
use windmill_common::worker::{to_raw_value, Connection};
use windmill_parser_sql::{parse_db_resource, parse_mssql_sig};
use windmill_queue::MiniPulledJob;
use windmill_queue::{append_logs, CanceledBy};
@@ -51,14 +51,14 @@ pub async fn do_mssql(
job: &MiniPulledJob,
client: &AuthedClient,
query: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
occupancy_metrics: &mut OccupancyMetrics,
job_dir: &str,
) -> error::Result<Box<RawValue>> {
let mssql_args = build_args_values(job, client, db).await?;
let mssql_args = build_args_values(job, client, conn).await?;
let inline_db_res_path = parse_db_resource(&query);
@@ -104,7 +104,7 @@ pub async fn do_mssql(
if readonly_intent {
let logs = format!("\nSetting ApplicationIntent to ReadOnly");
append_logs(&job.id, &job.workspace_id, logs, db).await;
append_logs(&job.id, &job.workspace_id, logs, conn).await;
}
// Handle authentication based on available credentials
@@ -222,7 +222,7 @@ pub async fn do_mssql(
let raw_result = run_future_with_polling_update_job_poller(
job.id,
job.timeout,
db,
conn,
mem_peak,
canceled_by,
result_f,
backend/windmill-worker/src/mysql_executor.rs
+6 -6
View File
@@ -11,7 +11,7 @@ use serde_json::{json, value::RawValue, Value};
use tokio::sync::Mutex;
use windmill_common::{
error::{to_anyhow, Error},
worker::to_raw_value,
worker::{to_raw_value, Connection},
};
use windmill_parser_sql::{
parse_db_resource, parse_mysql_sig, parse_sql_blocks, parse_sql_statement_named_params,
@@ -106,14 +106,14 @@ pub async fn do_mysql(
job: &MiniPulledJob,
client: &AuthedClient,
query: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
column_order: &mut Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> windmill_common::error::Result<Box<RawValue>> {
let job_args = build_args_values(job, client, db).await?;
let job_args = build_args_values(job, client, conn).await?;
let inline_db_res_path = parse_db_resource(&query);
@@ -234,8 +234,8 @@ pub async fn do_mysql(
}
let pool = mysql_async::Pool::new(opts);
let conn = pool.get_conn().await.map_err(to_anyhow)?;
let conn_a = Arc::new(Mutex::new(conn));
let mysql_conn = pool.get_conn().await.map_err(to_anyhow)?;
let conn_a = Arc::new(Mutex::new(mysql_conn));
let queries = parse_sql_blocks(query);
@@ -281,7 +281,7 @@ pub async fn do_mysql(
let result = run_future_with_polling_update_job_poller(
job.id,
job.timeout,
db,
conn,
mem_peak,
canceled_by,
result_f,
backend/windmill-worker/src/nu_executor.rs
+11 -8
View File
@@ -3,7 +3,10 @@ use std::{collections::HashMap, process::Stdio};
use itertools::Itertools;
use serde_json::value::RawValue;
use tokio::{fs::File, io::AsyncWriteExt, process::Command};
use windmill_common::{error::Error, worker::write_file};
use windmill_common::{
error::Error,
worker::{write_file, Connection},
};
use windmill_parser::Arg;
use windmill_parser_nu::parse_nu_signature;
use windmill_queue::{append_logs, CanceledBy, MiniPulledJob};
@@ -30,7 +33,7 @@ pub(crate) struct JobHandlerInput<'a> {
pub canceled_by: &'a mut Option<CanceledBy>,
pub client: &'a AuthedClient,
pub parent_runnable_path: Option<String>,
pub db: &'a sqlx::Pool<sqlx::Postgres>,
pub conn: &'a Connection,
pub envs: HashMap<String, String>,
pub inner_content: &'a str,
pub job: &'a MiniPulledJob,
@@ -52,7 +55,7 @@ pub async fn handle_nu_job<'a>(mut args: JobHandlerInput<'a>) -> Result<Box<RawV
// --- Handle relative ---
// --- Wrap and write to fs ---
{
create_args_and_out_file(&args.client, args.job, args.job_dir, args.db).await?;
create_args_and_out_file(&args.client, args.job, args.job_dir, args.conn).await?;
File::create(format!("{}/main.nu", args.job_dir))
.await?
.write_all(&wrap(args.inner_content)?.into_bytes())
@@ -217,7 +220,7 @@ async fn run<'a>(
canceled_by,
worker_name,
job,
db,
conn,
job_dir,
shared_mount,
client,
@@ -229,13 +232,13 @@ async fn run<'a>(
// plugins: Vec<&'a str>,
) -> Result<(), Error> {
let reserved_variables =
get_reserved_variables(job, &client.token, db, parent_runnable_path.clone()).await?;
get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone()).await?;
let child = if !cfg!(windows) && !*DISABLE_NSJAIL {
append_logs(
&job.id,
&job.workspace_id,
format!("\n\n--- ISOLATED NU CODE EXECUTION ---\n"),
db.clone(),
conn,
)
.await;
@@ -274,7 +277,7 @@ async fn run<'a>(
&job.id,
&job.workspace_id,
format!("\n\n--- NU CODE EXECUTION ---\n"),
db.clone(),
&conn,
)
.await;
@@ -323,7 +326,7 @@ async fn run<'a>(
};
handle_child::handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
backend/windmill-worker/src/oracledb_executor.rs
+6 -6
View File
@@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize};
use serde_json::{json, value::RawValue, Value};
use windmill_common::{
error::{to_anyhow, Error},
worker::to_raw_value,
worker::{to_raw_value, Connection},
};
use windmill_queue::MiniPulledJob;
@@ -296,7 +296,7 @@ pub async fn do_oracledb(
job: &MiniPulledJob,
client: &AuthedClient,
query: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
@@ -309,7 +309,7 @@ pub async fn do_oracledb(
"Oracle Database",
)?;
let job_args = build_args_values(job, client, db).await?;
let job_args = build_args_values(job, client, conn).await?;
let inline_db_res_path = parse_db_resource(&query);
@@ -354,14 +354,14 @@ pub async fn do_oracledb(
.init();
}
let conn = tokio::task::spawn_blocking(|| {
let oracle_conn = tokio::task::spawn_blocking(|| {
oracle::Connection::connect(database.user, database.password, database.database)
.map_err(|e| Error::ExecutionErr(e.to_string()))
})
.await
.map_err(to_anyhow)??;
let conn_a = Arc::new(std::sync::Mutex::new(conn));
let conn_a = Arc::new(std::sync::Mutex::new(oracle_conn));
let queries = parse_sql_blocks(&query);
@@ -396,7 +396,7 @@ pub async fn do_oracledb(
let result = run_future_with_polling_update_job_poller(
job.id,
job.timeout,
db,
conn,
mem_peak,
canceled_by,
result_f,
backend/windmill-worker/src/pg_executor.rs
+4 -4
View File
@@ -27,7 +27,7 @@ use tokio_postgres::{
use uuid::Uuid;
use windmill_common::error::to_anyhow;
use windmill_common::error::{self, Error};
use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
use windmill_common::worker::{to_raw_value, Connection, CLOUD_HOSTED};
use windmill_parser::{Arg, Typ};
use windmill_parser_sql::{
parse_db_resource, parse_pg_statement_arg_indices, parse_pgsql_sig, parse_sql_blocks,
@@ -161,14 +161,14 @@ pub async fn do_postgresql(
job: &MiniPulledJob,
client: &AuthedClient,
query: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
column_order: &mut Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<Box<RawValue>> {
let pg_args = build_args_values(job, client, db).await?;
let pg_args = build_args_values(job, client, conn).await?;
let inline_db_res_path = parse_db_resource(&query);
@@ -353,7 +353,7 @@ pub async fn do_postgresql(
let result = run_future_with_polling_update_job_poller(
job.id,
job.timeout,
db,
conn,
mem_peak,
canceled_by,
result_f,
backend/windmill-worker/src/php_executor.rs
+12 -11
View File
@@ -7,7 +7,7 @@ use tokio::{fs::File, io::AsyncReadExt, process::Command};
use uuid::Uuid;
use windmill_common::{
error::{self, to_anyhow, Result},
worker::write_file,
worker::{write_file, Connection},
};
use windmill_queue::MiniPulledJob;
@@ -67,7 +67,7 @@ pub async fn composer_install(
canceled_by: &mut Option<CanceledBy>,
job_id: &Uuid,
w_id: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
job_dir: &str,
worker_name: &str,
requirements: String,
@@ -94,7 +94,7 @@ pub async fn composer_install(
handle_child(
job_id,
db,
conn,
mem_peak,
canceled_by,
child_process,
@@ -139,7 +139,7 @@ pub async fn handle_php_job(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job: &MiniPulledJob,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
client: &AuthedClient,
parent_runnable_path: Option<String>,
job_dir: &str,
@@ -167,14 +167,14 @@ pub async fn handle_php_job(
let autoload_line = if let Some(composer_json) = composer_json {
let logs1 = "\n\n--- COMPOSER INSTALL ---\n".to_string();
append_logs(&job.id, &job.workspace_id, logs1, db).await;
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
composer_install(
mem_peak,
canceled_by,
&job.id,
&job.workspace_id,
db,
conn,
job_dir,
worker_name,
composer_json,
@@ -189,7 +189,7 @@ pub async fn handle_php_job(
let init_logs = "\n\n--- PHP CODE EXECUTION ---\n".to_string();
append_logs(&job.id, job.workspace_id.to_string(), init_logs, db).await;
append_logs(&job.id, job.workspace_id.to_string(), init_logs, conn).await;
let _ = write_file(job_dir, "main.php", inner_content)?;
@@ -263,12 +263,13 @@ try {{
let reserved_variables_args_out_f = async {
let args_and_out_f = async {
create_args_and_out_file(&client, job, job_dir, db).await?;
create_args_and_out_file(&client, job, job_dir, conn).await?;
Ok(()) as Result<()>
};
let reserved_variables_f = async {
let vars = get_reserved_variables(job, &client.token, db, parent_runnable_path.clone())
.await?;
let vars =
get_reserved_variables(job, &client.token, conn, parent_runnable_path.clone())
.await?;
Ok(vars) as Result<HashMap<String, String>>
};
let (_, reserved_variables) = tokio::try_join!(args_and_out_f, reserved_variables_f)?;
@@ -329,7 +330,7 @@ try {{
handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
backend/windmill-worker/src/python_executor.rs
+171 -118
View File
@@ -10,7 +10,6 @@ use anyhow::anyhow;
use itertools::Itertools;
use regex::Regex;
use serde_json::value::RawValue;
use sqlx::{Pool, Postgres};
use tokio::{
fs::{metadata, DirBuilder, File},
io::AsyncReadExt,
@@ -27,15 +26,16 @@ use windmill_common::{
Error::{self},
},
utils::calculate_hash,
worker::{copy_dir_recursively, pad_string, write_file, PythonAnnotations, WORKER_CONFIG},
DB,
worker::{
copy_dir_recursively, pad_string, write_file, Connection, PythonAnnotations, WORKER_CONFIG,
},
};
#[cfg(feature = "enterprise")]
use windmill_common::variables::get_secret_value_as_admin;
use std::env::var;
use windmill_queue::{append_logs, CanceledBy};
use windmill_queue::{append_logs, CanceledBy, PrecomputedAgentInfo};
lazy_static::lazy_static! {
static ref PYTHON_PATH: Option<String> = var("PYTHON_PATH").ok().map(|v| {
@@ -77,6 +77,7 @@ use crate::{
start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
worker_utils::ping_job_status,
AuthedClient, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, INSTANCE_PYTHON_VERSION, NSJAIL_PATH,
PATH_ENV, PIP_EXTRA_INDEX_URL, PIP_INDEX_URL, PROXY_ENVS, PY_INSTALL_DIR, TZ_ENV, UV_CACHE_DIR,
};
@@ -95,7 +96,7 @@ pub enum PyVersion {
}
impl PyVersion {
pub async fn from_instance_version(job_id: &Uuid, w_id: &str, db: &Pool<Postgres>) -> Self {
pub async fn from_instance_version(job_id: &Uuid, w_id: &str, conn: &Connection) -> Self {
let mut err = None;
let pyv = match INSTANCE_PYTHON_VERSION.read().await.clone() {
Some(v) => PyVersion::from_string_with_dots(&v).unwrap_or_else(|| {
@@ -108,7 +109,7 @@ impl PyVersion {
};
if let Some(msg) = err {
append_logs(job_id, w_id, &msg, db).await;
append_logs(job_id, w_id, &msg, conn).await;
tracing::error!(msg);
}
pyv
@@ -211,7 +212,7 @@ impl PyVersion {
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
db: &Pool<Postgres>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
@@ -221,7 +222,7 @@ impl PyVersion {
// }
let res = self
.get_python_inner(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics)
.get_python_inner(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await;
if let Err(ref e) = res {
@@ -235,7 +236,7 @@ impl PyVersion {
format!(
"\nError while getting python from uv, falling back to system python: {e:?}"
),
db,
conn,
)
.await;
}
@@ -246,7 +247,7 @@ impl PyVersion {
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
db: &Pool<Postgres>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
@@ -257,7 +258,7 @@ impl PyVersion {
if py_path.is_err() {
// Install it
if let Err(err) = self
.install_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics)
.install_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await
{
tracing::error!("Cannot install python: {err}");
@@ -283,13 +284,13 @@ impl PyVersion {
job_id: &Uuid,
mem_peak: &mut i32,
// canceled_by: &mut Option<CanceledBy>,
db: &Pool<Postgres>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<()> {
let v = self.to_string_with_dot();
append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), db).await;
append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), conn).await;
// Create dirs for newly installed python
// If we dont do this, NSJAIL will not be able to mount cache
// For the default version directory created during startup (main.rs)
@@ -337,10 +338,10 @@ impl PyVersion {
let child_process = start_child_process(child_cmd, "uv").await?;
append_logs(&job_id, &w_id, logs, db).await;
append_logs(&job_id, &w_id, logs, conn).await;
handle_child(
job_id,
db,
conn,
mem_peak,
&mut None,
child_process,
@@ -459,7 +460,7 @@ pub async fn uv_pip_compile(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job_dir: &str,
db: &Pool<Postgres>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
@@ -506,25 +507,27 @@ pub async fn uv_pip_compile(
let requirements = format!("# py{}\n{}", py_version.to_string_no_dot(), requirements);
#[cfg(feature = "enterprise")]
let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?;
let requirements = replace_pip_secret(conn, w_id, &requirements, worker_name, job_id).await?;
let req_hash = format!("py-{}", calculate_hash(&requirements));
if !no_cache {
if let Some(cached) = sqlx::query_scalar!(
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
// Python version is included in hash,
// hash will be the different for every python version
req_hash
)
.fetch_optional(db)
.await?
{
logs.push_str(&format!(
"\nFound cached resolution: {req_hash}, on python version: {}",
py_version.to_string_with_dot()
));
return Ok(cached);
if let Some(db) = conn.as_sql() {
if let Some(cached) = sqlx::query_scalar!(
"SELECT lockfile FROM pip_resolution_cache WHERE hash = $1",
// Python version is included in hash,
// hash will be the different for every python version
req_hash
)
.fetch_optional(db)
.await?
{
logs.push_str(&format!(
"\nFound cached resolution: {req_hash}, on python version: {}",
py_version.to_string_with_dot()
));
return Ok(cached);
}
}
}
@@ -535,7 +538,7 @@ pub async fn uv_pip_compile(
{
// Make sure we have python runtime installed
py_version
.get_python(job_id, mem_peak, db, worker_name, w_id, occupancy_metrics)
.get_python(job_id, mem_peak, conn, worker_name, w_id, occupancy_metrics)
.await?;
let mut args = vec![
@@ -631,10 +634,10 @@ pub async fn uv_pip_compile(
}
let child_process = start_child_process(child_cmd, uv_cmd).await?;
append_logs(&job_id, &w_id, logs, db).await;
append_logs(&job_id, &w_id, logs, conn).await;
handle_child(
job_id,
db,
conn,
mem_peak,
canceled_by,
child_process,
@@ -671,11 +674,13 @@ pub async fn uv_pip_compile(
.collect::<Vec<String>>()
.join("\n")
);
sqlx::query!(
if let Some(db) = conn.as_sql() {
sqlx::query!(
"INSERT INTO pip_resolution_cache (hash, lockfile, expiration) VALUES ($1, $2, now() + ('3 days')::interval) ON CONFLICT (hash) DO UPDATE SET lockfile = $2",
req_hash,
lockfile
).fetch_optional(db).await?;
}
Ok(lockfile)
}
@@ -711,7 +716,7 @@ async fn postinstall(
additional_python_paths: &mut Vec<String>,
job_dir: &str,
job: &MiniPulledJob,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
) -> windmill_common::error::Result<()> {
// It is guranteed that additional_python_paths only contains paths within windmill/cache/
// All other paths you would usually expect in PYTHONPATH are NOT included. These are added in downstream
@@ -772,7 +777,7 @@ async fn postinstall(
&job.id,
&job.workspace_id,
"\n\nCopying some packages from cache to job_dir...\n".to_string(),
db,
conn,
)
.await;
// Remove PATHs we just moved
@@ -789,13 +794,20 @@ async fn get_python_path(
job_id: &Uuid,
w_id: &str,
mem_peak: &mut i32,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> windmill_common::error::Result<String> {
let python_path = if let Some(python_path) = PYTHON_PATH.clone() {
python_path
} else if let Some(python_path) = py_version
.get_python(&job_id, mem_peak, db, worker_name, w_id, occupancy_metrics)
.get_python(
&job_id,
mem_peak,
conn,
worker_name,
w_id,
occupancy_metrics,
)
.await?
{
python_path
@@ -816,7 +828,7 @@ pub async fn handle_python_job(
job: &MiniPulledJob,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
client: &AuthedClient,
parent_runnable_path: Option<String>,
inner_content: &String,
@@ -825,6 +837,7 @@ pub async fn handle_python_job(
envs: HashMap<String, String>,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
precomputed_agent_info: Option<PrecomputedAgentInfo>,
) -> windmill_common::error::Result<Box<RawValue>> {
let script_path = crate::common::use_flow_root_path(job.runnable_path());
@@ -835,12 +848,13 @@ pub async fn handle_python_job(
&job.workspace_id,
&script_path,
&job.id,
db,
conn,
worker_name,
worker_dir,
mem_peak,
canceled_by,
&mut Some(occupancy_metrics),
precomputed_agent_info,
)
.await?;
@@ -852,13 +866,13 @@ pub async fn handle_python_job(
&job.id,
&job.workspace_id,
mem_peak,
db,
conn,
&mut Some(occupancy_metrics),
)
.await?;
if !no_postinstall {
if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, db).await {
if let Err(e) = postinstall(&mut additional_python_paths, job_dir, job, conn).await {
tracing::error!("Postinstall stage has failed. Reason: {e}");
}
tracing::debug!("Finished deps postinstall stage");
@@ -872,7 +886,7 @@ pub async fn handle_python_job(
"\n\n--- PYTHON ({}) CODE EXECUTION ---\n",
py_version.to_string_with_dot()
),
db,
conn,
)
.await;
}
@@ -901,7 +915,7 @@ pub async fn handle_python_job(
let apply_preprocessor = pre_spread.is_some();
create_args_and_out_file(&client, job, job_dir, db).await?;
create_args_and_out_file(&client, job, job_dir, conn).await?;
tracing::debug!("Finished preparing wrapper");
let preprocessor = if let Some(pre_spread) = pre_spread {
@@ -1004,7 +1018,7 @@ except BaseException as e:
tracing::debug!("Finished writing wrapper");
let mut reserved_variables =
get_reserved_variables(job, &client.token, db, parent_runnable_path).await?;
get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
// Add /tmp/windmill/cache/python_xyz/global-site-packages to PYTHONPATH.
// Usefull if certain wheels needs to be preinstalled before execution.
@@ -1129,7 +1143,7 @@ mount {{
handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
@@ -1353,43 +1367,47 @@ async fn prepare_wrapper(
#[cfg(feature = "enterprise")]
async fn replace_pip_secret(
db: &DB,
conn: &Connection,
w_id: &str,
req: &str,
worker_name: &str,
job_id: &Uuid,
) -> error::Result<String> {
if PIP_SECRET_VARIABLE.is_match(req) {
let mut joined = "".to_string();
for req in req.lines() {
let nreq = if PIP_SECRET_VARIABLE.is_match(req) {
let capture = PIP_SECRET_VARIABLE.captures(req);
let variable = capture.unwrap().get(1).unwrap().as_str();
if !variable.contains("/PIP_SECRET_") {
return Err(error::Error::internal_err(format!(
if let Some(db) = conn.as_sql() {
if PIP_SECRET_VARIABLE.is_match(req) {
let mut joined = "".to_string();
for req in req.lines() {
let nreq = if PIP_SECRET_VARIABLE.is_match(req) {
let capture = PIP_SECRET_VARIABLE.captures(req);
let variable = capture.unwrap().get(1).unwrap().as_str();
if !variable.contains("/PIP_SECRET_") {
return Err(error::Error::internal_err(format!(
"invalid secret variable in pip requirements, (last part of path ma): {}",
req
)));
}
let secret = get_secret_value_as_admin(db, w_id, variable).await?;
tracing::info!(
worker = %worker_name,
job_id = %job_id,
workspace_id = %w_id,
"found secret variable in pip requirements: {}",
req
);
PIP_SECRET_VARIABLE
.replace(req, secret.as_str())
.to_string()
} else {
req.to_string()
};
joined.push_str(&nreq);
joined.push_str("\n");
}
}
let secret = get_secret_value_as_admin(db, w_id, variable).await?;
tracing::info!(
worker = %worker_name,
job_id = %job_id,
workspace_id = %w_id,
"found secret variable in pip requirements: {}",
req
);
PIP_SECRET_VARIABLE
.replace(req, secret.as_str())
.to_string()
} else {
req.to_string()
};
joined.push_str(&nreq);
joined.push_str("\n");
}
Ok(joined)
Ok(joined)
} else {
Ok(req.to_string())
}
} else {
Ok(req.to_string())
}
@@ -1402,12 +1420,13 @@ async fn handle_python_deps(
w_id: &str,
script_path: &str,
job_id: &Uuid,
db: &DB,
conn: &Connection,
worker_name: &str,
worker_dir: &str,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
precomputed_agent_info: Option<PrecomputedAgentInfo>,
) -> error::Result<(PyVersion, Vec<String>)> {
create_dependencies_dir(job_dir).await;
@@ -1423,23 +1442,32 @@ async fn handle_python_deps(
let mut annotated_pyv = None;
let mut annotated_pyv_numeric = None;
let is_deployed = requirements_o.is_some();
let instance_pyv = PyVersion::from_instance_version(job_id, w_id, db).await;
let instance_pyv = PyVersion::from_instance_version(job_id, w_id, conn).await;
let annotations = windmill_common::worker::PythonAnnotations::parse(inner_content);
let requirements = match requirements_o {
Some(r) => r,
None => {
let mut already_visited = vec![];
requirements = windmill_parser_py_imports::parse_python_imports(
inner_content,
w_id,
script_path,
db,
&mut already_visited,
&mut annotated_pyv_numeric,
)
.await?
.join("\n");
requirements = match conn {
Connection::Sql(db) => windmill_parser_py_imports::parse_python_imports(
inner_content,
w_id,
script_path,
db,
&mut already_visited,
&mut annotated_pyv_numeric,
)
.await?
.join("\n"),
Connection::Http(_) => match precomputed_agent_info {
Some(PrecomputedAgentInfo::Python { py_version, requirements }) => {
annotated_pyv_numeric = py_version;
requirements.clone().unwrap_or_else(|| "".to_string())
}
_ => "".to_string(),
},
};
annotated_pyv = annotated_pyv_numeric.and_then(|v| PyVersion::from_numeric(v));
@@ -1450,7 +1478,7 @@ async fn handle_python_deps(
mem_peak,
canceled_by,
job_dir,
db,
conn,
worker_name,
w_id,
occupancy_metrics,
@@ -1491,7 +1519,7 @@ async fn handle_python_deps(
w_id,
mem_peak,
canceled_by,
db,
conn,
worker_name,
job_dir,
worker_dir,
@@ -1696,7 +1724,7 @@ pub async fn handle_python_reqs(
w_id: &str,
mem_peak: &mut i32,
_canceled_by: &mut Option<CanceledBy>,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
_worker_name: &str,
job_dir: &str,
worker_dir: &str,
@@ -1719,7 +1747,7 @@ pub async fn handle_python_reqs(
counter_arc: Arc<tokio::sync::Mutex<usize>>,
total_to_install: usize,
instant: std::time::Instant,
db: Pool<Postgres>,
conn: &Connection,
) {
#[cfg(not(all(feature = "enterprise", feature = "parquet", unix)))]
{
@@ -1748,7 +1776,7 @@ pub async fn handle_python_reqs(
if s3_push { " > (S3) " } else { "" },
instant.elapsed().as_millis(),
),
db,
conn,
)
.await;
// Drop lock, so next print success can fire
@@ -1810,7 +1838,7 @@ pub async fn handle_python_reqs(
&job_id,
w_id,
format!("\nenv deps from local cache: {}\n", in_cache.join(", ")),
db,
conn,
)
.await;
}
@@ -1824,7 +1852,7 @@ pub async fn handle_python_reqs(
let (_done_tx, mut done_rx) = tokio::sync::mpsc::channel::<()>(1);
let job_id_2 = job_id.clone();
let db_2 = db.clone();
let conn_2 = conn.clone();
let w_id_2 = w_id.to_string();
// Wheels to install
@@ -1874,9 +1902,12 @@ pub async fn handle_python_reqs(
*mem_peak_lock
};
// Notify server that we are still alive
// Detect if job has been canceled
let canceled = sqlx::query_scalar!(
let canceled = match conn_2 {
Connection::Sql(ref db) => {
sqlx::query_scalar!(
"UPDATE v2_job_runtime r SET
memory_peak = $1,
ping = now()
@@ -1885,17 +1916,25 @@ pub async fn handle_python_reqs(
RETURNING canceled_by IS NOT NULL AS \"canceled!\"",
mem_peak_actual,
job_id_2
)
.fetch_optional(&db_2)
.await
.unwrap_or_else(|e| {
tracing::error!(%e, "error updating job {job_id_2}: {e:#}");
Some(false)
})
.unwrap_or_else(|| {
// if the job is not in queue, it can only be in the completed_job so it is already complete
false
});
)
.fetch_optional(db)
.await
.unwrap_or_else(|e| {
tracing::error!(%e, "error updating job {job_id_2}: {e:#}");
Some(false)
})
.unwrap_or_else(|| {
// if the job is not in queue, it can only be in the completed_job so it is already complete
false
})
}
Connection::Http(_) => {
if let Err(e) = ping_job_status(&conn_2, &job_id_2, Some(mem_peak_actual), None).await {
tracing::error!(%e, "error pinging job {job_id_2}: {e:#}");
}
false
}
};
if canceled {
@@ -1952,7 +1991,7 @@ pub async fn handle_python_reqs(
parallel_limit
));
}
append_logs(&job_id, w_id, logs, db).await;
append_logs(&job_id, w_id, logs, conn).await;
}
let semaphore = Arc::new(Semaphore::new(parallel_limit));
@@ -1964,7 +2003,14 @@ pub async fn handle_python_reqs(
let total_time = std::time::Instant::now();
let py_path = py_version
.get_python(job_id, mem_peak, db, _worker_name, w_id, _occupancy_metrics)
.get_python(
job_id,
mem_peak,
conn,
_worker_name,
w_id,
_occupancy_metrics,
)
.await?;
let has_work = req_with_penv.len() > 0;
@@ -1988,7 +2034,7 @@ pub async fn handle_python_reqs(
"started setup python dependencies"
);
let db = db.clone();
let conn = conn.clone();
let job_id = job_id.clone();
let job_dir = job_dir.to_owned();
let w_id = w_id.to_owned();
@@ -2037,7 +2083,7 @@ pub async fn handle_python_reqs(
counter_arc,
total_to_install,
start,
db
&conn
).await;
pids.lock().await.get_mut(i).and_then(|e| e.take());
@@ -2076,7 +2122,7 @@ pub async fn handle_python_reqs(
format!(
"\nError while spawning proccess:\n{e}",
),
db,
&conn,
)
.await;
pids.lock().await.get_mut(i).and_then(|e| e.take());
@@ -2127,7 +2173,7 @@ pub async fn handle_python_reqs(
"\nError while installing {}:\n{stderr_buf}",
&req
),
db,
&conn,
)
.await;
pids.lock().await.get_mut(i).and_then(|e| e.take());
@@ -2164,7 +2210,7 @@ pub async fn handle_python_reqs(
counter_arc,
total_to_install,
start,
db, //
&conn, //
)
.await;
@@ -2218,7 +2264,13 @@ pub async fn handle_python_reqs(
if has_work {
let total_time = total_time.elapsed().as_millis();
append_logs(&job_id, w_id, format!("\nenv set in {}ms", total_time), db).await;
append_logs(
&job_id,
w_id,
format!("\nenv set in {}ms", total_time),
conn,
)
.await;
}
*mem_peak = *mem_peak_thread_safe.lock().await;
@@ -2285,7 +2337,7 @@ pub async fn start_worker(
let mut mem_peak: i32 = 0;
let mut canceled_by: Option<CanceledBy> = None;
let context = variables::get_reserved_variables(
db,
&Connection::Sql(db.clone()),
w_id,
&token,
"dedicated_worker@windmill.dev",
@@ -2312,12 +2364,13 @@ pub async fn start_worker(
w_id,
script_path,
&Uuid::nil(),
db,
&Connection::Sql(db.clone()),
worker_name,
job_dir,
&mut mem_peak,
&mut canceled_by,
&mut None,
None,
)
.await?;
@@ -2400,7 +2453,7 @@ for line in sys.stdin:
}
let reserved_variables = windmill_common::variables::get_reserved_variables(
db,
&Connection::Sql(db.clone()),
w_id,
token,
"dedicated_worker",
@@ -2442,7 +2495,7 @@ for line in sys.stdin:
&Uuid::nil(),
w_id,
&mut mem_peak,
db,
&Connection::Sql(db.clone()),
&mut None,
)
.await?;
backend/windmill-worker/src/result_processor.rs
+120 -86
View File
@@ -2,7 +2,7 @@
use opentelemetry::trace::FutureExt;
use serde::Serialize;
use sqlx::{types::Json, Pool, Postgres};
use sqlx::types::Json;
use std::{
collections::HashMap,
sync::{
@@ -21,35 +21,105 @@ use windmill_common::{
error::{self, Error},
jobs::JobKind,
utils::WarnAfterExt,
worker::{to_raw_value, WORKER_GROUP},
worker::{to_raw_value, Connection, WORKER_GROUP},
KillpillSender, DB,
};
#[cfg(feature = "benchmark")]
use crate::bench::{BenchmarkInfo, BenchmarkIter};
use windmill_queue::{append_logs, get_queued_job, CanceledBy, MiniPulledJob, WrappedError};
use windmill_queue::{
append_logs, get_queued_job, CanceledBy, JobCompleted, MiniPulledJob, WrappedError,
};
use serde_json::{json, value::RawValue};
use tokio::{
sync::mpsc::{Receiver, Sender},
task::JoinHandle,
};
use tokio::{sync::mpsc::Receiver, task::JoinHandle};
use windmill_queue::{add_completed_job, add_completed_job_error};
use crate::{
bash_executor::ANSI_ESCAPE_RE,
common::{read_result, save_in_cache},
common::{error_to_value, read_result, save_in_cache},
otel_ee::add_root_flow_job_to_otlp,
worker_flow::update_flow_status_after_job_completion,
AuthedClient, JobCompleted, JobCompletedSender, SameWorkerSender, SendResult, INIT_SCRIPT_TAG,
AuthedClient, JobCompletedSender, SameWorkerSender, SendResult, INIT_SCRIPT_TAG,
};
async fn process_jc(
jc: JobCompleted,
worker_name: &str,
base_internal_url: &str,
db: &DB,
worker_dir: &str,
same_worker_tx: &SameWorkerSender,
job_completed_sender: &JobCompletedSender,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) {
let success: bool = jc.success;
let span = tracing::span!(
tracing::Level::INFO,
"job_postprocessing",
job_id = %jc.job.id, root_job = field::Empty, workspace_id = %jc.job.workspace_id, worker = %worker_name,tag = %jc.job.tag,
// hostname = %hostname,
language = field::Empty,
script_path = field::Empty,
flow_step_id = field::Empty,
parent_job = field::Empty,
otel.name = field::Empty
);
let rj = if let Some(root_job) = jc.job.flow_innermost_root_job {
root_job
} else {
jc.job.id
};
windmill_common::otel_ee::set_span_parent(&span, &rj);
if let Some(lg) = jc.job.script_lang.as_ref() {
span.record("language", lg.as_str());
}
if let Some(step_id) = jc.job.flow_step_id.as_ref() {
span.record(
"otel.name",
format!("job_postprocessing {}", step_id).as_str(),
);
span.record("flow_step_id", step_id.as_str());
} else {
span.record("otel.name", "job postprocessing");
}
if let Some(parent_job) = jc.job.parent_job.as_ref() {
span.record("parent_job", parent_job.to_string().as_str());
}
if let Some(script_path) = jc.job.runnable_path.as_ref() {
span.record("script_path", script_path.as_str());
}
if let Some(root_job) = jc.job.flow_innermost_root_job.as_ref() {
span.record("root_job", root_job.to_string().as_str());
}
let root_job = handle_receive_completed_job(
jc,
&base_internal_url,
&db,
&worker_dir,
&same_worker_tx,
&worker_name,
job_completed_sender.clone(),
#[cfg(feature = "benchmark")]
bench,
)
.instrument(span)
.await;
if let Some(root_job) = root_job {
add_root_flow_job_to_otlp(&root_job, success);
}
}
pub fn start_background_processor(
mut job_completed_rx: Receiver<SendResult>,
job_completed_sender: Sender<SendResult>,
job_completed_sender: JobCompletedSender,
same_worker_queue_size: Arc<AtomicU16>,
job_completed_processor_is_done: Arc<AtomicBool>,
base_internal_url: String,
@@ -86,66 +156,19 @@ pub fn start_background_processor(
JobKind::Dependencies | JobKind::FlowDependencies
);
let success = jc.success;
let span = tracing::span!(
tracing::Level::INFO,
"job_postprocessing",
job_id = %jc.job.id, root_job = field::Empty, workspace_id = %jc.job.workspace_id, worker = %worker_name,tag = %jc.job.tag,
// hostname = %hostname,
language = field::Empty,
script_path = field::Empty,
flow_step_id = field::Empty,
parent_job = field::Empty,
otel.name = field::Empty
);
let rj = if let Some(root_job) = jc.job.flow_innermost_root_job {
root_job
} else {
jc.job.id
};
windmill_common::otel_ee::set_span_parent(&span, &rj);
if let Some(lg) = jc.job.script_lang.as_ref() {
span.record("language", lg.as_str());
}
if let Some(step_id) = jc.job.flow_step_id.as_ref() {
span.record(
"otel.name",
format!("job_postprocessing {}", step_id).as_str(),
);
span.record("flow_step_id", step_id.as_str());
} else {
span.record("otel.name", "job postprocessing");
}
if let Some(parent_job) = jc.job.parent_job.as_ref() {
span.record("parent_job", parent_job.to_string().as_str());
}
if let Some(script_path) = jc.job.runnable_path.as_ref() {
span.record("script_path", script_path.as_str());
}
if let Some(root_job) = jc.job.flow_innermost_root_job.as_ref() {
span.record("root_job", root_job.to_string().as_str());
}
let root_job = handle_receive_completed_job(
process_jc(
jc,
&worker_name,
&base_internal_url,
&db,
&worker_dir,
&same_worker_tx,
&worker_name,
job_completed_sender.clone(),
&job_completed_sender,
#[cfg(feature = "benchmark")]
&mut bench,
)
.instrument(span)
.await;
if let Some(root_job) = root_job {
add_root_flow_job_to_otlp(&root_job, success);
}
if is_init_script_and_failure {
tracing::error!("init script errored, exiting");
killpill_tx.send();
@@ -250,7 +273,7 @@ async fn send_job_completed(
duration,
};
job_completed_tx
.send(jc)
.send_job(jc)
.with_context(windmill_common::otel_ee::otel_ctx())
.await
.expect("send job completed")
@@ -267,20 +290,22 @@ pub async fn process_result(
token: &str,
column_order: Option<Vec<String>>,
new_args: Option<HashMap<String, Box<RawValue>>>,
db: &DB,
conn: &Connection,
duration: Option<i64>,
) -> error::Result<bool> {
match result {
Ok(r) => {
// Update script args to preprocessed args
if let Some(preprocessed_args) = new_args {
sqlx::query!(
"UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2",
Json(preprocessed_args) as Json<HashMap<String, Box<RawValue>>>,
job.id
)
.execute(db)
.await?;
if let Connection::Sql(db) = conn {
if let Some(preprocessed_args) = new_args {
sqlx::query!(
"UPDATE v2_job SET args = $1, preprocessed = TRUE WHERE id = $2",
Json(preprocessed_args) as Json<HashMap<String, Box<RawValue>>>,
job.id
)
.execute(db)
.await?;
}
}
send_job_completed(
@@ -307,18 +332,30 @@ pub async fn process_result(
if res.as_ref().is_some_and(|x| !x.get().is_empty()) {
res.unwrap()
} else {
let last_10_log_lines = sqlx::query_scalar!(
match conn {
Connection::Sql(db) => {
let last_10_log_lines = sqlx::query_scalar!(
"SELECT right(logs, 600) FROM job_logs WHERE job_id = $1 AND workspace_id = $2 ORDER BY created_at DESC LIMIT 1",
&job.id,
&job.workspace_id
).fetch_one(db).await.ok().flatten().unwrap_or("".to_string());
let log_lines = last_10_log_lines
.split("CODE EXECUTION ---")
.last()
.unwrap_or(&last_10_log_lines);
let log_lines = last_10_log_lines
.split("CODE EXECUTION ---")
.last()
.unwrap_or(&last_10_log_lines);
extract_error_value(&program, log_lines, i, job.flow_step_id.clone())
extract_error_value(
&program,
log_lines,
i,
job.flow_step_id.clone(),
)
}
Connection::Http(_) => {
to_raw_value(&"See logs for more details".to_string())
}
}
}
}
err @ _ => to_raw_value(&SerializedError {
@@ -355,7 +392,7 @@ pub async fn handle_receive_completed_job(
worker_dir: &str,
same_worker_tx: &SameWorkerSender,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
job_completed_tx: JobCompletedSender,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> Option<Arc<MiniPulledJob>> {
let token = jc.token.clone();
@@ -422,7 +459,7 @@ pub async fn process_completed_job(
worker_dir: &str,
same_worker_tx: SameWorkerSender,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
job_completed_tx: JobCompletedSender,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> error::Result<Option<Arc<MiniPulledJob>>> {
if success {
@@ -543,7 +580,7 @@ pub async fn process_completed_job(
#[tracing::instrument(name = "job_error", level = "info", skip_all, fields(job_id = %job.id))]
pub async fn handle_job_error(
db: &Pool<Postgres>,
db: &DB,
client: &AuthedClient,
job: &MiniPulledJob,
mem_peak: i32,
@@ -553,20 +590,17 @@ pub async fn handle_job_error(
same_worker_tx: SameWorkerSender,
worker_dir: &str,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
job_completed_tx: JobCompletedSender,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) {
let err = match err {
Error::JsonErr(err) => err,
_ => json!({"message": err.to_string(), "name": "InternalErr"}),
};
let err = error_to_value(err);
let update_job_future = || async {
append_logs(
&job.id,
&job.workspace_id,
format!("Unexpected error during job execution:\n{err:#?}"),
db,
&db.into(),
)
.await;
add_completed_job_error(
@@ -626,7 +660,7 @@ pub async fn handle_job_error(
&parent_job.id,
&job.workspace_id,
format!("Unexpected error during flow job error handling:\n{err}"),
db,
&db.into(),
)
.await;
let _ = add_completed_job_error(
backend/windmill-worker/src/rust_executor.rs
+14 -14
View File
@@ -8,7 +8,7 @@ use tokio::{fs::File, io::AsyncReadExt, process::Command};
use windmill_common::{
error::{self, Error},
utils::calculate_hash,
worker::{save_cache, write_file},
worker::{save_cache, write_file, Connection},
};
use windmill_queue::MiniPulledJob;
use windmill_queue::{append_logs, CanceledBy};
@@ -127,7 +127,7 @@ pub async fn generate_cargo_lockfile(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job_dir: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut OccupancyMetrics,
@@ -153,7 +153,7 @@ pub async fn generate_cargo_lockfile(
let gen_lockfile_process = start_child_process(gen_lockfile_cmd, CARGO_PATH.as_str()).await?;
handle_child(
job_id,
db,
conn,
mem_peak,
canceled_by,
gen_lockfile_process,
@@ -180,7 +180,7 @@ pub async fn build_rust_crate(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job_dir: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
worker_name: &str,
w_id: &str,
base_internal_url: &str,
@@ -216,7 +216,7 @@ pub async fn build_rust_crate(
let build_rust_process = start_child_process(build_rust_cmd, CARGO_PATH.as_str()).await?;
handle_child(
job_id,
db,
conn,
mem_peak,
canceled_by,
build_rust_process,
@@ -230,7 +230,7 @@ pub async fn build_rust_crate(
None,
)
.await?;
append_logs(job_id, w_id, "\n\n", db).await;
append_logs(job_id, w_id, "\n\n", conn).await;
tokio::fs::copy(
&format!("{job_dir}/target/release/main"),
@@ -279,7 +279,7 @@ pub async fn handle_rust_job(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job: &MiniPulledJob,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
client: &AuthedClient,
parent_runnable_path: Option<String>,
inner_content: &str,
@@ -314,11 +314,11 @@ pub async fn handle_rust_job(
))
})?;
create_args_and_out_file(client, job, job_dir, db).await?;
create_args_and_out_file(client, job, job_dir, conn).await?;
cache_logs
} else {
let logs1 = format!("{cache_logs}\n\n--- CARGO BUILD ---\n");
append_logs(&job.id, &job.workspace_id, logs1, db).await;
append_logs(&job.id, &job.workspace_id, logs1, conn).await;
gen_cargo_crate(inner_content, job_dir)?;
@@ -328,14 +328,14 @@ pub async fn handle_rust_job(
}
}
create_args_and_out_file(client, job, job_dir, db).await?;
create_args_and_out_file(client, job, job_dir, conn).await?;
build_rust_crate(
&job.id,
mem_peak,
canceled_by,
job_dir,
db,
conn,
worker_name,
&job.workspace_id,
base_internal_url,
@@ -346,10 +346,10 @@ pub async fn handle_rust_job(
};
let logs2 = format!("{cache_logs}\n\n--- RUST CODE EXECUTION ---\n");
append_logs(&job.id, &job.workspace_id, logs2, db).await;
append_logs(&job.id, &job.workspace_id, logs2, conn).await;
let reserved_variables =
get_reserved_variables(job, &client.token, db, parent_runnable_path).await?;
get_reserved_variables(job, &client.token, conn, parent_runnable_path).await?;
let child = if !*DISABLE_NSJAIL {
let _ = write_file(
@@ -400,7 +400,7 @@ pub async fn handle_rust_job(
};
handle_child(
&job.id,
db,
conn,
mem_peak,
canceled_by,
child,
backend/windmill-worker/src/snowflake_executor.rs
+6 -5
View File
@@ -9,6 +9,7 @@ use serde_json::{json, value::RawValue, Value};
use sha2::{Digest, Sha256};
use std::collections::HashMap;
use windmill_common::error::to_anyhow;
use windmill_common::worker::Connection;
use windmill_common::{error::Error, worker::to_raw_value};
use windmill_parser_sql::{parse_db_resource, parse_snowflake_sig, parse_sql_blocks};
@@ -249,14 +250,14 @@ pub async fn do_snowflake(
job: &MiniPulledJob,
client: &AuthedClient,
query: &str,
db: &sqlx::Pool<sqlx::Postgres>,
conn: &Connection,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
column_order: &mut Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> windmill_common::error::Result<Box<RawValue>> {
let snowflake_args = build_args_values(job, client, db).await?;
let snowflake_args = build_args_values(job, client, conn).await?;
let inline_db_res_path = parse_db_resource(&query);
@@ -362,7 +363,7 @@ pub async fn do_snowflake(
json!(database.database.unwrap().to_uppercase()),
);
}
let timeout = resolve_job_timeout(&db, &job.workspace_id, job.id, job.timeout)
let timeout = resolve_job_timeout(&conn, &job.workspace_id, job.id, job.timeout)
.await
.0
.as_secs();
@@ -371,7 +372,7 @@ pub async fn do_snowflake(
let queries = parse_sql_blocks(query);
let (timeout_duration, _, _) =
resolve_job_timeout(&db, &job.workspace_id, job.id, job.timeout).await;
resolve_job_timeout(&conn, &job.workspace_id, job.id, job.timeout).await;
let http_client = build_http_client(timeout_duration)?;
@@ -424,7 +425,7 @@ pub async fn do_snowflake(
let r = run_future_with_polling_update_job_poller(
job.id,
job.timeout,
db,
conn,
mem_peak,
canceled_by,
result_f.map_err(to_anyhow),
backend/windmill-worker/src/worker.rs
+644 -742
View File
File diff suppressed because it is too large Load Diff
backend/windmill-worker/src/worker_flow.rs
+17 -110
View File
@@ -16,8 +16,7 @@ use crate::bench::BenchmarkIter;
use crate::common::{cached_result_path, save_in_cache};
use crate::js_eval::{eval_timeout, IdContext};
use crate::{
AuthedClient, PreviousResult, SameWorkerPayload, SameWorkerSender, SendResult, JOB_TOKEN,
KEEP_JOB_DIR,
AuthedClient, JobCompletedSender, PreviousResult, SameWorkerSender, SendResult, KEEP_JOB_DIR,
};
use anyhow::Context;
use futures::TryFutureExt;
@@ -27,7 +26,6 @@ use serde_json::value::RawValue;
use serde_json::{json, Value};
use sqlx::types::Json;
use sqlx::{FromRow, Postgres, Transaction};
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use uuid::Uuid;
use windmill_common::add_time;
@@ -54,11 +52,12 @@ use windmill_common::{
},
flows::{FlowModule, FlowModuleValue, FlowValue, InputTransform, Retry, Suspend},
};
use windmill_queue::flow_status::Step;
use windmill_queue::schedule::get_schedule_opt;
use windmill_queue::{
add_completed_job, add_completed_job_error, append_logs, get_mini_pulled_job,
handle_maybe_scheduled_job, CanceledBy, MiniPulledJob, PushArgs, PushIsolationLevel,
WrappedError,
SameWorkerPayload, WrappedError,
};
type DB = sqlx::Pool<sqlx::Postgres>;
@@ -81,7 +80,7 @@ pub async fn update_flow_status_after_job_completion(
worker_dir: &str,
stop_early_override: Option<bool>,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
job_completed_tx: JobCompletedSender,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> error::Result<Option<Arc<MiniPulledJob>>> {
// this is manual tailrecursion because async_recursion blows up the stack
@@ -200,7 +199,7 @@ pub async fn update_flow_status_after_job_completion_internal(
stop_early_override: Option<bool>,
skip_error_handler: bool,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
job_completed_tx: JobCompletedSender,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> error::Result<UpdateFlowStatusAfterJobCompletion> {
add_time!(bench, "update flow status internal START");
@@ -1050,7 +1049,7 @@ pub async fn update_flow_status_after_job_completion_internal(
} else {
"Flow job completed with error\n".to_string()
};
append_logs(&flow_job.id, w_id, logs, db).await;
append_logs(&flow_job.id, w_id, logs, &db.into()).await;
}
#[cfg(feature = "enterprise")]
if flow_job.parent_job.is_none() {
@@ -1168,7 +1167,7 @@ pub async fn update_flow_status_after_job_completion_internal(
&flow_job.id,
w_id,
format!("Unexpected error during flow chaining:\n{:#?}", e),
db,
&db.into(),
)
.await;
let _ = add_completed_job_error(db, &flow_job, 0, None, e, worker_name, true, None)
@@ -1388,102 +1387,6 @@ async fn compute_bool_from_expr(
}
}
pub async fn update_flow_status_in_progress(
db: &DB,
_w_id: &str,
flow: Uuid,
job_in_progress: Uuid,
) -> error::Result<Step> {
let step = get_step_of_flow_status(db, flow).await?;
match step {
Step::Step(step) => {
sqlx::query!(
"UPDATE v2_job_status SET
flow_status = jsonb_set(
jsonb_set(flow_status, ARRAY['modules', $3::INTEGER::TEXT, 'job'], to_jsonb($1::UUID::TEXT)),
ARRAY['modules', $3::INTEGER::TEXT, 'type'],
to_jsonb('InProgress'::text)
)
WHERE id = $2",
job_in_progress,
flow,
step as i32
)
.execute(db)
.await?;
}
Step::PreprocessorStep => {
sqlx::query!(
"UPDATE v2_job_status SET
flow_status = jsonb_set(
jsonb_set(flow_status, ARRAY['preprocessor_module', 'job'], to_jsonb($1::UUID::TEXT)),
ARRAY['preprocessor_module', 'type'],
to_jsonb('InProgress'::text)
)
WHERE id = $2",
job_in_progress,
flow
)
.execute(db)
.await?;
}
Step::FailureStep => {
sqlx::query!(
"UPDATE v2_job_status SET
flow_status = jsonb_set(
jsonb_set(flow_status, ARRAY['failure_module', 'job'], to_jsonb($1::UUID::TEXT)),
ARRAY['failure_module', 'type'],
to_jsonb('InProgress'::text)
)
WHERE id = $2",
job_in_progress,
flow
)
.execute(db)
.await?;
}
}
Ok(step)
}
#[derive(Debug, Copy, Clone)]
pub enum Step {
Step(usize),
PreprocessorStep,
FailureStep,
}
impl Step {
fn from_i32_and_len(step: i32, len: usize) -> Self {
if step < 0 {
Step::PreprocessorStep
} else if (step as usize) < len {
Step::Step(step as usize)
} else {
Step::FailureStep
}
}
}
#[instrument(level = "trace", skip_all)]
pub async fn get_step_of_flow_status(db: &DB, id: Uuid) -> error::Result<Step> {
let r = sqlx::query!(
"SELECT (flow_status->'step')::integer as step, jsonb_array_length(flow_status->'modules') as len
FROM v2_job_status WHERE id = $1",
id
)
.fetch_one(db)
.await
.map_err(|e| Error::internal_err(format!("fetching step flow status: {e:#}")))?;
if let Some(step) = r.step {
Ok(Step::from_i32_and_len(step, r.len.unwrap_or(0) as usize))
} else {
Err(Error::internal_err("step is null".to_string()))
}
}
/// resumes should be in order of timestamp ascending, so that more recent are at the end
#[instrument(level = "trace", skip_all)]
async fn transform_input(
@@ -1548,7 +1451,7 @@ pub async fn handle_flow(
last_result: Option<Arc<Box<RawValue>>>,
same_worker_tx: SameWorkerSender,
worker_dir: &str,
job_completed_tx: Sender<SendResult>,
job_completed_tx: JobCompletedSender,
worker_name: &str,
) -> anyhow::Result<()> {
let flow = flow_data.value();
@@ -1669,7 +1572,7 @@ async fn push_next_flow_job(
last_job_result: Option<Arc<Box<RawValue>>>,
same_worker_tx: SameWorkerSender,
worker_dir: &str,
job_completed_tx: Sender<SendResult>,
job_completed_tx: JobCompletedSender,
worker_name: &str,
) -> error::Result<Option<PushNextFlowJobRec>> {
let job_root = flow_job
@@ -2105,7 +2008,13 @@ async fn push_next_flow_job(
let result: Value = json!({ "error": {"message": logs, "name": error_name}});
append_logs(&flow_job.id, &flow_job.workspace_id, logs.clone(), db).await;
append_logs(
&flow_job.id,
&flow_job.workspace_id,
logs.clone(),
&db.into(),
)
.await;
job_completed_tx
.send(SendResult::UpdateFlow {
@@ -2644,7 +2553,7 @@ async fn push_next_flow_job(
};
// forward root job permissions to the new job
let job_perms: Option<Authed> = if JOB_TOKEN.is_none() {
let job_perms: Option<Authed> = {
if let Some(root_job) = &flow_job
.flow_innermost_root_job
.or_else(|| Some(flow_job.id))
@@ -2661,8 +2570,6 @@ async fn push_next_flow_job(
} else {
None
}
} else {
None
};
tracing::debug!(id = %flow_job.id, root_id = %job_root, "computed perms for job {i} of {len}");
backend/windmill-worker/src/worker_lockfiles.rs
+24 -17
View File
@@ -17,7 +17,7 @@ use windmill_common::jobs::JobPayload;
use windmill_common::scripts::ScriptHash;
#[cfg(feature = "python")]
use windmill_common::worker::PythonAnnotations;
use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file};
use windmill_common::worker::{to_raw_value, to_raw_value_owned, write_file, Connection};
use windmill_common::{
apps::AppScriptId,
@@ -83,7 +83,7 @@ pub async fn update_script_dependency_map(
)
.await?;
tx.commit().await?;
append_logs(job_id, w_id, logs, db).await;
append_logs(job_id, w_id, logs, &db.into()).await;
}
Ok(())
}
@@ -226,7 +226,7 @@ pub async fn handle_dependency_job(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
job_dir: &str,
db: &sqlx::Pool<sqlx::Postgres>,
db: &DB,
worker_name: &str,
worker_dir: &str,
base_internal_url: &str,
@@ -265,7 +265,7 @@ pub async fn handle_dependency_job(
// - A saved script `hash` in the `script_hash` column.
// - Preview raw lock and code in the `queue` or `job` table.
let script_data = &match job.runnable_id {
Some(hash) => match cache::script::fetch(db, hash).await {
Some(hash) => match cache::script::fetch(&Connection::from(db.clone()), hash).await {
Ok(d) => Cow::Owned(d.0),
Err(e) => {
let logs2 = sqlx::query_scalar!(
@@ -1121,7 +1121,7 @@ async fn lock_modules<'c>(
Some(e.id.clone()),
)
.await?;
append_logs(&job.id, &job.workspace_id, logs, db).await;
append_logs(&job.id, &job.workspace_id, logs, &db.into()).await;
}
if language == ScriptLang::Bun || language == ScriptLang::Bunnative {
@@ -1513,7 +1513,7 @@ async fn lock_modules_app(
.await;
match new_lock {
Ok(new_lock) => {
append_logs(&job.id, &job.workspace_id, logs, db).await;
append_logs(&job.id, &job.workspace_id, logs, &db.into()).await;
let anns =
windmill_common::worker::TypeScriptAnnotations::parse(
&content,
@@ -1761,7 +1761,7 @@ async fn python_dep(
let final_version = annotated_pyv_numeric
.and_then(|pyv| PyVersion::from_numeric(pyv))
.unwrap_or(PyVersion::from_instance_version(job_id, w_id, db).await);
.unwrap_or(PyVersion::from_instance_version(job_id, w_id, &db.into()).await);
let req: std::result::Result<String, Error> = uv_pip_compile(
job_id,
@@ -1769,7 +1769,7 @@ async fn python_dep(
mem_peak,
canceled_by,
job_dir,
db,
&db.into(),
worker_name,
w_id,
occupancy_metrics,
@@ -1785,7 +1785,7 @@ async fn python_dep(
w_id,
mem_peak,
canceled_by,
db,
&Connection::Sql(db.clone()),
worker_name,
job_dir,
worker_dir,
@@ -1918,7 +1918,7 @@ async fn capture_dependency_job(
mem_peak,
canceled_by,
job_dir,
db,
&db.into(),
false,
false,
false,
@@ -1940,7 +1940,7 @@ async fn capture_dependency_job(
mem_peak,
canceled_by,
job_dir,
Some(db),
Some(&db.into()),
w_id,
worker_name,
base_internal_url,
@@ -1960,7 +1960,7 @@ async fn capture_dependency_job(
canceled_by,
job_id,
w_id,
Some(db),
Some(&db.into()),
token,
script_path,
job_dir,
@@ -1983,7 +1983,7 @@ async fn capture_dependency_job(
script_path,
job_id,
w_id,
Some(db.clone()),
Some(&db),
&job_dir,
base_internal_url,
worker_name,
@@ -2020,7 +2020,7 @@ async fn capture_dependency_job(
canceled_by,
job_id,
w_id,
db,
&Connection::Sql(db.clone()),
job_dir,
worker_name,
reqs,
@@ -2049,7 +2049,7 @@ async fn capture_dependency_job(
mem_peak,
canceled_by,
job_dir,
db,
&Connection::Sql(db.clone()),
worker_name,
w_id,
occupancy_metrics,
@@ -2072,7 +2072,7 @@ async fn capture_dependency_job(
mem_peak,
canceled_by,
job_dir,
db,
&Connection::Sql(db.clone()),
worker_name,
w_id,
occupancy_metrics,
@@ -2087,7 +2087,14 @@ async fn capture_dependency_job(
));
}
resolve(job_id, job_raw_code, job_dir, db, w_id).await
resolve(
job_id,
job_raw_code,
job_dir,
&Connection::Sql(db.clone()),
w_id,
)
.await
}
// for related places search: ADD_NEW_LANG
_ => Ok("".to_owned()),
backend/windmill-worker/src/worker_utils.rs
+322
View File
@@ -0,0 +1,322 @@
use backon::{BackoffBuilder, ConstantBuilder, Retryable};
use tracing::Instrument;
use uuid::Uuid;
use windmill_common::{
agent_workers::{PingJobStatus, PingJobStatusResponse},
worker::{
get_memory, get_vcpus, get_windmill_memory_usage, get_worker_memory_usage,
insert_ping_query, update_job_ping_query, update_worker_ping_from_job_query,
update_worker_ping_main_loop_query, Connection, Ping, PingType, WORKER_CONFIG,
WORKER_GROUP,
},
KillpillSender,
};
use crate::{
agent_workers::UPDATE_PING_URL,
common::{OccupancyMetrics, OccupancyResult},
};
pub(crate) async fn update_worker_ping_full(
conn: &Connection,
read_cgroups: bool,
jobs_executed: i32,
worker_name: &str,
hostname: &str,
occupancy_metrics: &mut OccupancyMetrics,
killpill_tx: &KillpillSender,
) {
let tags = WORKER_CONFIG.read().await.worker_tags.clone();
let memory_usage = get_worker_memory_usage();
let wm_memory_usage = get_windmill_memory_usage();
let (vcpus, memory) = if read_cgroups {
(get_vcpus(), get_memory())
} else {
(None, None)
};
let OccupancyResult {
occupancy_rate,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
} = occupancy_metrics.update_occupancy_metrics();
if let Err(e) = (|| {
update_worker_ping_full_inner(
conn,
jobs_executed,
&worker_name,
&tags,
memory_usage,
wm_memory_usage,
vcpus,
memory,
occupancy_rate,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
)
})
.retry(
ConstantBuilder::default()
.with_delay(std::time::Duration::from_secs(2))
.with_max_times(10)
.build(),
)
.notify(|err, dur| {
tracing::error!(
worker = %worker_name, hostname = %hostname,
"retrying updating worker ping in {dur:#?}, err: {err:#?}"
);
})
.sleep(tokio::time::sleep)
.await
{
tracing::error!(
worker = %worker_name, hostname = %hostname,
"failed to update worker ping, exiting: {}", e);
killpill_tx.send();
}
tracing::info!(
worker = %worker_name, hostname = %hostname,
"ping update, memory: container={}MB, windmill={}MB",
memory_usage.unwrap_or_default() / (1024 * 1024),
wm_memory_usage.unwrap_or_default() / (1024 * 1024)
);
}
async fn update_worker_ping_full_inner(
conn: &Connection,
jobs_executed: i32,
worker_name: &str,
tags: &[String],
memory_usage: Option<i64>,
wm_memory_usage: Option<i64>,
vcpus: Option<i64>,
memory: Option<i64>,
occupancy_rate: f32,
occupancy_rate_15s: Option<f32>,
occupancy_rate_5m: Option<f32>,
occupancy_rate_30m: Option<f32>,
) -> anyhow::Result<()> {
match conn {
Connection::Sql(db) => {
update_worker_ping_main_loop_query(
worker_name,
tags,
vcpus,
memory,
Some(jobs_executed),
Some(occupancy_rate),
memory_usage,
wm_memory_usage,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
db,
)
.await?;
}
Connection::Http(client) => {
client
.post::<_, ()>(
UPDATE_PING_URL,
&Ping {
last_job_executed: None,
last_job_workspace_id: None,
worker_instance: None,
ip: None,
tags: Some(tags.to_vec()),
dw: None,
jobs_executed: Some(jobs_executed),
occupancy_rate: Some(occupancy_rate),
occupancy_rate_15s: Some(occupancy_rate_15s.unwrap_or(0.0)),
occupancy_rate_5m: Some(occupancy_rate_5m.unwrap_or(0.0)),
occupancy_rate_30m: Some(occupancy_rate_30m.unwrap_or(0.0)),
version: None,
vcpus: vcpus,
memory: memory,
memory_usage: get_worker_memory_usage(),
wm_memory_usage: get_windmill_memory_usage(),
ping_type: PingType::MainLoop,
},
)
.await?;
}
}
Ok(())
}
pub async fn insert_ping(
worker_instance: &str,
worker_name: &str,
ip: &str,
db: &Connection,
) -> anyhow::Result<()> {
let (tags, dw) = {
let wc = WORKER_CONFIG.read().await.clone();
(
wc.worker_tags,
wc.dedicated_worker
.as_ref()
.map(|x| format!("{}:{}", x.workspace_id, x.path)),
)
};
let vcpus = get_vcpus();
let memory = get_memory();
match db {
Connection::Sql(db) => {
insert_ping_query(
worker_instance,
worker_name,
WORKER_GROUP.as_str(),
ip,
tags.as_slice(),
dw,
windmill_common::utils::GIT_VERSION,
vcpus,
memory,
db,
)
.await?;
}
Connection::Http(client) => {
client
.post::<_, ()>(
UPDATE_PING_URL,
&Ping {
last_job_executed: None,
last_job_workspace_id: None,
worker_instance: Some(worker_instance.to_string()),
ip: Some(ip.to_string()),
tags: Some(tags.to_vec()),
dw: dw,
jobs_executed: None,
occupancy_rate: None,
occupancy_rate_15s: None,
occupancy_rate_5m: None,
occupancy_rate_30m: None,
version: Some(windmill_common::utils::GIT_VERSION.to_string()),
vcpus: vcpus,
memory: memory,
memory_usage: get_worker_memory_usage(),
wm_memory_usage: get_windmill_memory_usage(),
ping_type: PingType::Initial,
},
)
.await?;
}
}
Ok(())
}
pub async fn update_worker_ping_from_job(
conn: &Connection,
job_id: &Uuid,
w_id: &str,
worker_name: &str,
memory_usage: Option<i64>,
wm_memory_usage: Option<i64>,
occupancy: Option<OccupancyResult>,
) -> anyhow::Result<()> {
let occupancy_rate = occupancy.as_ref().map(|x| x.occupancy_rate);
let occupancy_rate_15s = occupancy.as_ref().and_then(|x| x.occupancy_rate_15s);
let occupancy_rate_5m = occupancy.as_ref().and_then(|x| x.occupancy_rate_5m);
let occupancy_rate_30m = occupancy.as_ref().and_then(|x| x.occupancy_rate_30m);
match conn.clone() {
Connection::Sql(ref db) => {
update_worker_ping_from_job_query(
job_id,
w_id,
worker_name,
memory_usage,
wm_memory_usage,
occupancy_rate,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
db,
)
.await?;
}
Connection::Http(client) => {
client
.post::<Ping, ()>(
UPDATE_PING_URL,
&Ping {
last_job_executed: Some(job_id.clone()),
last_job_workspace_id: Some(w_id.to_string()),
ping_type: PingType::Job,
worker_instance: None,
ip: None,
tags: None,
dw: None,
version: None,
vcpus: None,
memory: None,
memory_usage: memory_usage,
wm_memory_usage: wm_memory_usage,
jobs_executed: None,
occupancy_rate: occupancy_rate,
occupancy_rate_15s: occupancy_rate_15s,
occupancy_rate_5m: occupancy_rate_5m,
occupancy_rate_30m: occupancy_rate_30m,
},
)
.await?;
}
}
Ok(())
}
pub async fn ping_job_status(
conn: &Connection,
job_id: &Uuid,
mem_peak: Option<i32>,
current_mem: Option<i32>,
) -> anyhow::Result<PingJobStatusResponse> {
match conn {
Connection::Sql(ref db) => update_job_ping_query(job_id, db, mem_peak).await,
Connection::Http(client) => {
client
.post(
&format!("/api/agent_workers/ping_job_status/{}", job_id),
&PingJobStatus { mem_peak, current_mem },
)
.await
}
}
}
pub(crate) async fn queue_vacuum(conn: &Connection, worker_name: &str, hostname: &str) {
match conn {
Connection::Sql(db) => {
let db2 = db.clone();
let current_span = tracing::Span::current();
let worker_name = worker_name.to_string();
let hostname = hostname.to_string();
tokio::task::spawn(
(async move {
tracing::info!(worker = %worker_name, hostname = %hostname, "vacuuming queue");
if let Err(e) = sqlx::query!("VACUUM v2_job_queue, v2_job_runtime, v2_job_status")
.execute(&db2)
.await
{
tracing::error!(worker = %worker_name, hostname = %hostname, "failed to vacuum queue: {}", e);
}
tracing::info!(worker = %worker_name, hostname = %hostname, "vacuumed queue");
})
.instrument(current_span),
);
}
Connection::Http(_) => {
// do nothing in http mode
()
}
}
}
frontend/src/lib/components/HttpAgentWorkerDrawer.svelte
+118
View File
@@ -0,0 +1,118 @@
<script lang="ts">
import { AgentWorkersService } from '$lib/gen'
import { sendUserToast } from '$lib/toast'
import { Copy } from 'lucide-svelte'
import { Alert } from './common'
import Section from './Section.svelte'
import TagsToListenTo from './TagsToListenTo.svelte'
import { enterpriseLicense, superadmin } from '$lib/stores'
import CollapseLink from './CollapseLink.svelte'
export let customTags: string[] | undefined
let selectedTags: string[] = !$enterpriseLicense ? ['agent_test'] : []
let workerGroup: string = 'agent'
let token: string = ''
$: selectedTags && selectedTags.length > 0 && $superadmin && workerGroup && refreshToken()
async function refreshToken() {
try {
const newToken = await AgentWorkersService.createAgentToken({
requestBody: {
worker_group: workerGroup,
tags: selectedTags,
exp: Math.floor(Date.now() / 1000) + 60 * 60 * 24 * 365 * 3 // 3 years
}
})
token = newToken
} catch (error) {
sendUserToast('Error creating agent token: ' + error.toString(), true)
}
}
</script>
<div class="flex flex-col gap-y-4">
<Alert type="info" title="HTTP agent workers "
>Use HTTP agent workers only when the workers need to be deployed remotely OR with only HTTP
connectivity OR in untrusted environments. HTTP agent workers have more latency and less
capabilities than normal workers.</Alert
>
<Section
label="Worker group"
tooltip="This is only used to give a name prefix to the agent worker and to group workers in the workers page, no worker group config is passed to an agent worker."
>
<input class="max-w-md" type="text" bind:value={workerGroup} />
</Section>
<Section label="Tags to listen to" eeOnly>
{#if !$enterpriseLicense}
<div class="text-sm text-secondary mb-2 max-w-md">
Agent workers are only available in the enterprise edition. For evaluation purposes, you can
only use the tag `agent_test` tag and it is limited to 100 jobs.
</div>
{/if}
<TagsToListenTo disabled={!$enterpriseLicense} bind:worker_tags={selectedTags} {customTags} />
</Section>
<Section label="Generated JWT token" primary>
<div class="relative max-w-md">
<input
on:click|preventDefault|stopPropagation|capture={() => {
navigator.clipboard.writeText(token)
sendUserToast('Copied to clipboard')
}}
placeholder="Select tags to generate a jwt token"
type="text"
disabled
value={token}
class="pr-8 text-sm text-secondary"
/>
<button
class="absolute right-2 top-1/2 -translate-y-1/2 text-primary"
on:click|preventDefault|stopPropagation={() => {
navigator.clipboard.writeText(token)
sendUserToast('Copied to clipboard')
}}
>
<Copy size={16} />
</button>
</div>
<div class="text-sm text-secondary mt-2 mb-12 max-w-md">
Pass the env variables:
<ul class="my-1">
<li>MODE=agent</li>
<li>AGENT_TOKEN={'"<token above>"'}</li>
<li>BASE_INTERNAL_URL={'"<base internal url>"'}</li>
</ul>
to a worker to have it act as an HTTP agent worker. INIT_SCRIPT, if needed, must be passed as an
env variable.
<p class="mt-4">
Remember to have at least one normal worker that listens to the tags `flow` and `dependency`
(or `flow-$workspace` and `dependency-$workspace` if using workspace specific default tags)
to have flow and dependency job being runnable as agent workers can't run dependency jobs
nor can run the flow state machine (but can run the subjobs within them).
</p>
</div>
<CollapseLink text="Automate JWT token generation" small>
<div class="text-xs text-secondary">
Use the following api endpoint with a superadmin bearer token:
<code class="text-primary"> POST /api/agent_workers/create_agent_token </code>
with body:
<pre>
<code class="text-primary">
{`{
"worker_group": "agent",
"tags": ["tag1", "tag2"],
"exp": 1717334400 // 3 years from now
}`}
</code>
</pre>
JSON response will be the JWT token.
</div>
</CollapseLink>
</Section>
</div>
frontend/src/lib/components/TagsToListenTo.svelte
+86
View File
@@ -0,0 +1,86 @@
<script lang="ts">
import { X } from 'lucide-svelte'
import { Button } from './common'
import { Plus } from 'lucide-svelte'
import { superadmin } from '$lib/stores'
import { createEventDispatcher } from 'svelte'
import AutoComplete from 'simple-svelte-autocomplete'
import { defaultTags, nativeTags } from './worker_group'
const dispatch = createEventDispatcher()
export let worker_tags: string[] = []
export let customTags: string[] = []
export let disabled = false
let newTag = ''
let createdTags: string[] = []
</script>
<div class="flex gap-3 gap-y-2 flex-wrap pb-2">
{#if worker_tags?.length == 0}
<div class="text-xs text-secondary">No tags selected</div>
{/if}
{#each worker_tags as tag}
<div class="flex gap-0.5 items-center"
><div class="text-2xs p-1 rounded border text-primary">{tag}</div>
{#if $superadmin && !disabled}
<button
class={'z-10 rounded-full p-1 duration-200 hover:bg-gray-200'}
aria-label="Remove item"
on:click|preventDefault|stopPropagation={() => {
worker_tags = worker_tags?.filter((t) => t != tag) ?? []
dispatch('dirty')
dispatch('deletePriorityTag', tag)
}}
>
<X size={12} />
</button>
{/if}</div
>
{/each}
</div>
{#if $superadmin}
<div class="max-w-md">
<AutoComplete
noInputStyles
items={[...(customTags ?? []), ...createdTags, ...defaultTags, ...nativeTags].filter(
(x) => !worker_tags?.includes(x)
)}
{disabled}
bind:selectedItem={newTag}
hideArrow={true}
inputClassName={'flex !font-gray-600 !font-primary !bg-surface-primary"'}
dropdownClassName="!text-sm !py-2 !rounded-sm !border-gray-200 !border !shadow-md"
className="w-full !font-gray-600 !font-primary !bg-surface-primary"
onFocus={() => {
dispatch('focus')
}}
create
onCreate={(c) => {
createdTags.push(c)
createdTags = [...createdTags]
return c
}}
createText="Press enter to use this tag"
/>
<div class="mt-1"></div>
<div class="flex">
<Button
variant="contained"
color="blue"
size="xs"
startIcon={{ icon: Plus }}
disabled={disabled || newTag == '' || worker_tags?.includes(newTag)}
on:click={() => {
worker_tags = [...(worker_tags ?? []), newTag.replaceAll(' ', '_')]
newTag = ''
dispatch('dirty')
}}
>
Add tag
</Button>
</div>
</div>
{/if}
frontend/src/lib/components/WorkerGroup.svelte
+150 -242
View File
@@ -18,8 +18,9 @@
import AutoComplete from 'simple-svelte-autocomplete'
import YAML from 'yaml'
import Toggle from './Toggle.svelte'
import type { AutoscalingConfig } from './worker_group'
import { defaultTags, nativeTags, type AutoscalingConfig } from './worker_group'
import AutoscalingConfigEditor from './AutoscalingConfigEditor.svelte'
import TagsToListenTo from './TagsToListenTo.svelte'
export let name: string
export let config:
@@ -77,10 +78,10 @@
? config
: {
worker_tags: []
}
}
: {
worker_tags: []
}
}
if (nconfig.priority_tags === undefined) {
nconfig.priority_tags = new Map<string, number>()
}
@@ -109,35 +110,6 @@
value: string | undefined
}[] = []
const defaultTags = [
'deno',
'python3',
'go',
'bash',
'powershell',
'dependency',
'flow',
'other',
'bun',
'php',
'rust',
'ansible',
'csharp',
'nu',
'java'
// for related places search: ADD_NEW_LANG
]
const nativeTags = [
'nativets',
'postgresql',
'mysql',
'graphql',
'snowflake',
'mssql',
'bigquery',
'oracledb'
]
const aws_env_vars_preset = [
'AWS_REGION',
'AWS_DEFAULT_REGION',
@@ -153,7 +125,6 @@
'SSL_CERT_DIR'
]
let newTag: string = ''
$: selected = nconfig?.dedicated_worker != undefined ? 'dedicated' : 'normal'
$: {
selectedPriorityTags = []
@@ -185,8 +156,6 @@
let openClean = false
let drawer: Drawer
let createdTags: string[] = []
</script>
<ConfirmationModal
@@ -246,7 +215,7 @@
Workers can still have their WORKER_TAGS, INIT_SCRIPT and WHITELIST_ENVS passed as env.
Dedicated workers are an enterprise only feature.
</Alert>
<div class="pb-4"></div>
<div class="pb-4" />
{/if}
<ToggleButtonGroup
@@ -285,216 +254,155 @@
</ToggleButtonGroup>
{#if selected == 'normal'}
<Section label="Tags to listen to">
{#if config?.worker_tags != undefined}
<div class="flex gap-3 gap-y-2 flex-wrap pb-2">
{#each config.worker_tags as tag}
<div class="flex gap-0.5 items-center"
><div class="text-2xs p-1 rounded border text-primary">{tag}</div>
{#if $superadmin}
<button
class={'z-10 rounded-full p-1 duration-200 hover:bg-gray-200'}
aria-label="Remove item"
on:click|preventDefault|stopPropagation={() => {
if (nconfig != undefined) {
dirty = true
nconfig.worker_tags = nconfig?.worker_tags?.filter((t) => t != tag) ?? []
if (nconfig.priority_tags) {
delete nconfig.priority_tags[tag]
}
selectedPriorityTags = selectedPriorityTags.filter((t) => t != tag) ?? []
}
}}
>
<X size={12} />
</button>
{/if}</div
>
{/each}
</div>
{#if $superadmin}
<div class="max-w-md">
<AutoComplete
noInputStyles
items={[
...(customTags ?? []),
...createdTags,
...defaultTags,
...nativeTags
].filter((x) => !nconfig?.worker_tags?.includes(x))}
bind:selectedItem={newTag}
hideArrow={true}
inputClassName={'flex !font-gray-600 !font-primary !bg-surface-primary"'}
dropdownClassName="!text-sm !py-2 !rounded-sm !border-gray-200 !border !shadow-md"
className="w-full !font-gray-600 !font-primary !bg-surface-primary"
onFocus={() => {
dispatch('focus')
}}
create
onCreate={(c) => {
createdTags.push(c)
createdTags = [...createdTags]
return c
}}
createText="Press enter to use this non-predefined value"
/>
{#if nconfig?.worker_tags != undefined}
<TagsToListenTo
on:dirty={() => {
dirty = true
}}
on:deletePriorityTag={(e) => {
const tag = e.detail
if (nconfig.priority_tags) {
delete nconfig.priority_tags[tag]
}
selectedPriorityTags = selectedPriorityTags.filter((t) => t != tag) ?? []
}}
bind:worker_tags={nconfig.worker_tags}
{customTags}
/>
<div class="mt-1"></div>
<div class="flex">
<Button
variant="contained"
color="blue"
size="xs"
startIcon={{ icon: Plus }}
disabled={newTag == '' || nconfig.worker_tags?.includes(newTag)}
on:click={() => {
if (nconfig != undefined) {
nconfig.worker_tags = [
...(nconfig?.worker_tags ?? []),
newTag.replaceAll(' ', '_')
]
newTag = ''
<div class="flex flex-wrap mt-2 items-center gap-1 pt-2">
<Button
variant="contained"
color="light"
size="xs"
on:click={() => {
if (nconfig != undefined) {
nconfig.worker_tags =
defaultTagPerWorkspace && workspaceTag
? defaultTags.concat(nativeTags).map((nt) => `${nt}-${workspaceTag}`)
: defaultTags.concat(nativeTags)
dirty = true
}
}}
>
Reset to all tags <Tooltip
>{(defaultTagPerWorkspace && workspaceTag
? defaultTags.concat(nativeTags).map((nt) => `${nt}-${workspaceTag}`)
: defaultTags.concat(nativeTags)
).join(', ')}</Tooltip
>
</Button>
<Button
variant="contained"
color="light"
size="xs"
on:click={() => {
if (nconfig != undefined) {
nconfig.worker_tags =
defaultTagPerWorkspace && workspaceTag
? defaultTags.map((nt) => `${nt}-${workspaceTag}`)
: defaultTags
dirty = true
}
}}
>
Reset to all tags minus native ones <Tooltip
>{(defaultTagPerWorkspace
? defaultTags.map((nt) => `${nt}-${workspaceTag}`)
: defaultTags
).join(', ')}</Tooltip
>
</Button>
<Button
variant="contained"
color="light"
size="xs"
on:click={() => {
if (nconfig != undefined) {
nconfig.worker_tags =
defaultTagPerWorkspace && workspaceTag
? nativeTags.map((nt) => `${nt}-${workspaceTag}`)
: nativeTags
dirty = true
}
}}
>
Reset to native tags <Tooltip
>{(defaultTagPerWorkspace && workspaceTag
? nativeTags.map((nt) => `${nt}-${workspaceTag}`)
: nativeTags
).join(', ')}</Tooltip
>
</Button>
{#if defaultTagPerWorkspace}
<AutoComplete
bind:selectedItem={workspaceTag}
noInputStyles
hideArrow={true}
items={workspaces.map((w) => w.id)}
inputClassName={'flex !font-gray-600 !font-primary !bg-surface-primary'}
dropdownClassName="!text-sm !py-2 !rounded-sm !border-gray-200 !border !shadow-md"
className="!font-gray-600 !font-primary !bg-surface-primary"
create
onCreate={(c) => c}
placeholder="Workspace ID"
/>
{/if}
</div>
<div class="max-w mt-2 items-center gap-1 pt-2">
{#if nconfig?.worker_tags !== undefined && nconfig?.worker_tags.length > 0}
<Label label="High-priority tags">
<svelte:fragment slot="header">
<Tooltip>
Jobs with the following high-priority tags will be picked up in priority by this
worker.
{#if !enterpriseLicense}
This is a feature only available in enterprise edition.
{/if}
</Tooltip>
</svelte:fragment>
<Multiselect
outerDivClass="text-secondary !bg-surface-disabled !border-0"
disabled={!$enterpriseLicense}
bind:selected={selectedPriorityTags}
on:change={(e) => {
if (e.detail.type === 'add') {
if (nconfig.priority_tags) {
if (e.detail.option && typeof e.detail.option !== 'object') {
nconfig.priority_tags[e.detail.option] = 100
}
}
dirty = true
} else if (e.detail.type === 'remove') {
if (nconfig.priority_tags) {
if (e.detail.option && typeof e.detail.option !== 'object') {
delete nconfig.priority_tags[e.detail.option]
}
}
dirty = true
} else if (e.detail.type === 'removeAll') {
nconfig.priority_tags = undefined
dirty = true
} else {
console.error(
`Priority tags multiselect - unknown event type: '${e.detail.type}'`
)
}
}}
>
Add tag
</Button>
</div>
</div>
<div class="flex flex-wrap mt-2 items-center gap-1 pt-2">
<Button
variant="contained"
color="light"
size="xs"
on:click={() => {
if (nconfig != undefined) {
nconfig.worker_tags =
defaultTagPerWorkspace && workspaceTag
? defaultTags.concat(nativeTags).map((nt) => `${nt}-${workspaceTag}`)
: defaultTags.concat(nativeTags)
dirty = true
}
}}
>
Reset to all tags <Tooltip
>{(defaultTagPerWorkspace && workspaceTag
? defaultTags.concat(nativeTags).map((nt) => `${nt}-${workspaceTag}`)
: defaultTags.concat(nativeTags)
).join(', ')}</Tooltip
>
</Button>
<Button
variant="contained"
color="light"
size="xs"
on:click={() => {
if (nconfig != undefined) {
nconfig.worker_tags =
defaultTagPerWorkspace && workspaceTag
? defaultTags.map((nt) => `${nt}-${workspaceTag}`)
: defaultTags
dirty = true
}
}}
>
Reset to all tags minus native ones <Tooltip
>{(defaultTagPerWorkspace
? defaultTags.map((nt) => `${nt}-${workspaceTag}`)
: defaultTags
).join(', ')}</Tooltip
>
</Button>
<Button
variant="contained"
color="light"
size="xs"
on:click={() => {
if (nconfig != undefined) {
nconfig.worker_tags =
defaultTagPerWorkspace && workspaceTag
? nativeTags.map((nt) => `${nt}-${workspaceTag}`)
: nativeTags
dirty = true
}
}}
>
Reset to native tags <Tooltip
>{(defaultTagPerWorkspace && workspaceTag
? nativeTags.map((nt) => `${nt}-${workspaceTag}`)
: nativeTags
).join(', ')}</Tooltip
>
</Button>
{#if defaultTagPerWorkspace}
<AutoComplete
bind:selectedItem={workspaceTag}
noInputStyles
hideArrow={true}
items={workspaces.map((w) => w.id)}
inputClassName={'flex !font-gray-600 !font-primary !bg-surface-primary'}
dropdownClassName="!text-sm !py-2 !rounded-sm !border-gray-200 !border !shadow-md"
className="!font-gray-600 !font-primary !bg-surface-primary"
create
onCreate={(c) => c}
placeholder="Workspace ID"
options={nconfig?.worker_tags}
selectedOptionsDraggable={false}
ulOptionsClass={'!bg-surface-secondary'}
placeholder="High priority tags"
/>
{/if}
</div>
<div class="max-w mt-2 items-center gap-1 pt-2">
{#if nconfig?.worker_tags !== undefined && nconfig?.worker_tags.length > 0}
<Label label="High-priority tags">
<svelte:fragment slot="header">
<Tooltip>
Jobs with the following high-priority tags will be picked up in priority by
this worker.
{#if !enterpriseLicense}
This is a feature only available in enterprise edition.
{/if}
</Tooltip>
</svelte:fragment>
<Multiselect
outerDivClass="text-secondary !bg-surface-disabled !border-0"
disabled={!$enterpriseLicense}
bind:selected={selectedPriorityTags}
on:change={(e) => {
if (e.detail.type === 'add') {
if (nconfig.priority_tags) {
if (e.detail.option && typeof e.detail.option !== 'object') {
nconfig.priority_tags[e.detail.option] = 100
}
}
dirty = true
} else if (e.detail.type === 'remove') {
if (nconfig.priority_tags) {
if (e.detail.option && typeof e.detail.option !== 'object') {
delete nconfig.priority_tags[e.detail.option]
}
}
dirty = true
} else if (e.detail.type === 'removeAll') {
nconfig.priority_tags = undefined
dirty = true
} else {
console.error(
`Priority tags multiselect - unknown event type: '${e.detail.type}'`
)
}
}}
options={nconfig?.worker_tags}
selectedOptionsDraggable={false}
ulOptionsClass={'!bg-surface-secondary'}
placeholder="High priority tags"
/>
</Label>
{/if}
</div>
{/if}
</Label>
{/if}
</div>
{/if}
</Section>
{#if nconfig !== undefined}
<div class="mt-8"></div>
<div class="mt-8" />
<Section label="Alerts" tooltip="Alert is sent to the configured critical error channels">
<Toggle
size="sm"
@@ -557,7 +465,7 @@
{/if}
{/if}
<div class="mt-8"></div>
<div class="mt-8" />
<Section
label="Python runtime settings"
collapsable={true}
@@ -668,7 +576,7 @@
</div>
</Section>
<div class="mt-8"></div>
<div class="mt-8" />
<Section
label="Environment variables passed to jobs"
@@ -808,7 +716,7 @@
</div>
{/if}
</Section>
<div class="mt-8"></div>
<div class="mt-8" />
<Section label="Autoscaling" collapsable>
<div slot="header" class="ml-4 flex flex-row gap-2 items-center">
@@ -823,7 +731,7 @@
bind:config={nconfig.autoscaling}
/>
</Section>
<div class="mt-8"></div>
<div class="mt-8" />
<Section
label="Init script"
frontend/src/lib/components/apps/editor/AppEditorHeader.svelte
+1
View File
@@ -62,6 +62,7 @@
import ToggleButtonGroup from '$lib/components/common/toggleButton-v2/ToggleButtonGroup.svelte'
import ToggleButton from '$lib/components/common/toggleButton-v2/ToggleButton.svelte'
import Tooltip from '$lib/components/Tooltip.svelte'
import { Sha256 } from '@aws-crypto/sha256-js'
import { sendUserToast } from '$lib/toast'
frontend/src/lib/components/worker_group.ts
+29
View File
@@ -37,3 +37,32 @@ export type AutoscaleScript = {
export type AutoscaleDryRun = {
type: 'dryrun'
}
export const defaultTags = [
'deno',
'python3',
'go',
'bash',
'powershell',
'dependency',
'flow',
'other',
'bun',
'php',
'rust',
'ansible',
'csharp',
'nu',
'java'
// KJQXZ
]
export const nativeTags = [
'nativets',
'postgresql',
'mysql',
'graphql',
'snowflake',
'mssql',
'bigquery',
'oracledb'
]
frontend/src/routes/(root)/(logged)/workers/+page.svelte
+23 -2
View File
@@ -32,6 +32,7 @@
import YAML from 'yaml'
import { DEFAULT_TAGS_WORKSPACES_SETTING } from '$lib/consts'
import AutoscalingEvents from '$lib/components/AutoscalingEvents.svelte'
import HttpAgentWorkerDrawer from '$lib/components/HttpAgentWorkerDrawer.svelte'
let workers: WorkerPing[] | undefined = undefined
let workerGroups: Record<string, any> | undefined = undefined
@@ -256,6 +257,8 @@
return Math.ceil(occupancy_rate * 100) + '%'
}
let newHttpAgentWorkerDrawer: Drawer | undefined = undefined
</script>
{#if $superadmin}
@@ -279,6 +282,15 @@
</DrawerContent>
</Drawer>
<Drawer bind:this={newHttpAgentWorkerDrawer} size="800px">
<DrawerContent
title="New HTTP agent worker"
on:close={() => newHttpAgentWorkerDrawer?.toggleDrawer?.()}
>
<HttpAgentWorkerDrawer {customTags} />
</DrawerContent>
</Drawer>
{#if $userStore?.operator && $workspaceStore && !$userWorkspaces.find((_) => _.id === $workspaceStore)?.operator_settings?.workers}
<div class="bg-red-100 border-l-4 border-red-600 text-orange-700 p-4 m-4 mt-12" role="alert">
<p class="font-bold">Unauthorized</p>
@@ -353,7 +365,16 @@
<div></div>
{#if $superadmin}
<div class="flex flex-row items-center">
<div class="flex flex-row gap-4 items-center">
<Button
size="sm"
color="light"
variant="border"
startIcon={{ icon: Plus }}
on:click={() => {
newHttpAgentWorkerDrawer?.toggleDrawer?.()
}}>New agent worker</Button
>
<Popover
floatingConfig={{ strategy: 'absolute', placement: 'bottom-end' }}
containerClasses="border rounded-lg shadow-lg p-4 bg-surface"
@@ -390,7 +411,7 @@
importConfigDrawer?.toggleDrawer?.()
}
}
]
]
: undefined}
>
<span class="hidden md:block"