starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-28 08:02:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into fr/flow-triggers-v0

Browse Source
This commit is contained in:
Faton Ramadani
2024-09-30 11:41:31 +02:00
committed by GitHub
parent 2f25c243c0 cc1a75e114
commit 46b25b851e
106 changed files with 3800 additions and 2084 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CHANGELOG.md
+30
View File
@@ -1,5 +1,35 @@
# Changelog
## [1.402.3](https://github.com/windmill-labs/windmill/compare/v1.402.2...v1.402.3) (2024-09-30)
### Bug Fixes
* improve allowed domains setting for sso ([24f4a7c](https://github.com/windmill-labs/windmill/commit/24f4a7caaafa93f51669dcf44a3dca09d5b228bb))
## [1.402.2](https://github.com/windmill-labs/windmill/compare/v1.402.1...v1.402.2) (2024-09-28)
### Bug Fixes
* make form properties disablable ([0779d47](https://github.com/windmill-labs/windmill/commit/0779d47c1d39626d11bd3769cd787cb036df0a94))
## [1.402.1](https://github.com/windmill-labs/windmill/compare/v1.402.0...v1.402.1) (2024-09-28)
### Bug Fixes
* allow preprocessor to write to args.json on nsjail ([#4455](https://github.com/windmill-labs/windmill/issues/4455)) ([0b9ec83](https://github.com/windmill-labs/windmill/commit/0b9ec83036e2a1d0773b4ec5856f907b383e9323))
* **frontend:** Fix flow graph bg in dark mode on chrome ([#4454](https://github.com/windmill-labs/windmill/issues/4454)) ([6956a3a](https://github.com/windmill-labs/windmill/commit/6956a3a2ba6d189528cb34ab05f7137fdf4f840b))
* improve suspend_first behavior and frequency ([b5e226b](https://github.com/windmill-labs/windmill/commit/b5e226b977e6d24ebd28bc1e7c867cb4888f77b2))
## [1.402.0](https://github.com/windmill-labs/windmill/compare/v1.401.0...v1.402.0) (2024-09-26)
### Features
* **cli:** add queues, workers and worker-groups commands ([#4439](https://github.com/windmill-labs/windmill/issues/4439)) ([9f91b19](https://github.com/windmill-labs/windmill/commit/9f91b1995a98c9e096c6e599c4d5a8d5ea499ada))
## [1.401.0](https://github.com/windmill-labs/windmill/compare/v1.400.0...v1.401.0) (2024-09-25)
backend/.sqlx/query-1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe.json
Generated
+25
View File
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int4",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Int4",
"Uuid",
"Text",
"Jsonb"
]
},
"nullable": [
null
]
},
"hash": "1e188d8e427cab25dbe18aa900260e26e644a9d939e74a8317c4a09335f110fe"
}
backend/.sqlx/query-3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Bool"
}
],
"parameters": {
"Left": [
"Text",
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "3e539fef054ad31bc1736e27276087775a721a6ee7ae35b03fd4ce3563ea3838"
}
backend/.sqlx/query-9bf41c3161a02b7d0731c4e1d79519cef5255f5df1b759af3aa4985bb64313e5.json → backend/.sqlx/query-41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO metrics (id, value)\n VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n ORDER BY priority DESC NULLS LAST, scheduled_for, created_at LIMIT 1)))",
"query": "INSERT INTO metrics (id, value)\n VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)\n FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval\n ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))",
"describe": {
"columns": [],
"parameters": {
@@ -11,5 +11,5 @@
},
"nullable": []
},
"hash": "9bf41c3161a02b7d0731c4e1d79519cef5255f5df1b759af3aa4985bb64313e5"
"hash": "41f68f4ce5bed783cf69e42da115e9ad2c9fcbd75f55817b2114c04207f66e4a"
}
backend/.sqlx/query-8375c1efeb1e2a2d2803052a2899bf70f4a6434eb91b4b05b9fb8420beae26af.json → backend/.sqlx/query-6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab.json
Generated
+23 -5
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3",
"query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,\n CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, \n custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3",
"describe": {
"columns": [
{
@@ -65,21 +65,36 @@
},
{
"ordinal": 12,
"name": "occupancy_rate_15s",
"type_info": "Float4"
},
{
"ordinal": 13,
"name": "occupancy_rate_5m",
"type_info": "Float4"
},
{
"ordinal": 14,
"name": "occupancy_rate_30m",
"type_info": "Float4"
},
{
"ordinal": 15,
"name": "memory",
"type_info": "Int8"
},
{
"ordinal": 13,
"ordinal": 16,
"name": "vcpus",
"type_info": "Int8"
},
{
"ordinal": 14,
"ordinal": 17,
"name": "memory_usage",
"type_info": "Int8"
},
{
"ordinal": 15,
"ordinal": 18,
"name": "wm_memory_usage",
"type_info": "Int8"
}
@@ -108,8 +123,11 @@
true,
true,
true,
true,
true,
true,
true
]
},
"hash": "8375c1efeb1e2a2d2803052a2899bf70f4a6434eb91b4b05b9fb8420beae26af"
"hash": "6a497334c98bfaf70be44fced572a1cc0dde4141aa4c5002765a95432d0101ab"
}
backend/.sqlx/query-6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46.json
Generated
+23
View File
@@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int4",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Int4",
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "6e7f234267fbb4720b29f288fba82c1df21ba601ac0989e175f834c569962d46"
}
backend/.sqlx/query-6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561.json
Generated
+73
View File
@@ -0,0 +1,73 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, raw_flow, flow_status) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 FROM generate_series(1, 1))",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8",
"Varchar",
{
"Custom": {
"name": "job_kind",
"kind": {
"Enum": [
"script",
"preview",
"flow",
"dependencies",
"flowpreview",
"script_hub",
"identity",
"flowdependencies",
"http",
"graphql",
"postgresql",
"noop",
"appdependencies",
"deploymentcallback",
"singlescriptflow"
]
}
}
},
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible"
]
}
}
},
"Varchar",
"Varchar",
"Varchar",
"Varchar",
"Timestamptz",
"Varchar",
"Jsonb",
"Jsonb"
]
},
"nullable": []
},
"hash": "6f4817fad2739a11d89b6704edf62c3c267ca336a8b6bec5b29d4409030ed561"
}
backend/.sqlx/query-a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c.json → backend/.sqlx/query-777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb)\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
"describe": {
"columns": [
{
@@ -19,5 +19,5 @@
null
]
},
"hash": "a94dbd1f7aab20682548471c5cc06c7a496edac36e3de537b00b76ad93c7556c"
"hash": "777190559e27c8c8fb6718b0a0c1d7db9b956abd88b94db3948f2c579c3826d0"
}
backend/.sqlx/query-804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf.json
Generated
+45
View File
@@ -0,0 +1,45 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Text",
"Varchar",
"Varchar",
"Int8",
{
"Custom": {
"name": "script_lang",
"kind": {
"Enum": [
"python3",
"deno",
"go",
"bash",
"postgresql",
"nativets",
"bun",
"mysql",
"bigquery",
"snowflake",
"graphql",
"powershell",
"mssql",
"php",
"bunnative",
"rust",
"ansible"
]
}
}
},
"Varchar",
"Varchar"
]
},
"nullable": []
},
"hash": "804fc11e35f4afc0db194b6fe2594f91df7e588d4d2431bc85f4d8734920c8bf"
}
backend/.sqlx/query-9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51.json
Generated
-21
View File
@@ -1,21 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus), memory = COALESCE($8, memory) WHERE worker = $6",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int4",
"TextArray",
"Float4",
"Int8",
"Int8",
"Text",
"Int8",
"Int8"
]
},
"nullable": []
},
"hash": "9cf96fa6364b7f34dc83719b4a0e97e8494393c29f7c8d915aa54da7ab7eed51"
}
backend/.sqlx/query-a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb.json
Generated
+24
View File
@@ -0,0 +1,24 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2,\n occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, vcpus = COALESCE($7, vcpus),\n memory = COALESCE($8, memory), occupancy_rate_15s = $9, occupancy_rate_5m = $10, occupancy_rate_30m = $11 WHERE worker = $6",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int4",
"TextArray",
"Float4",
"Int8",
"Int8",
"Text",
"Int8",
"Int8",
"Float4",
"Float4",
"Float4"
]
},
"nullable": []
},
"hash": "a439552f74ed0ba305e3d9cb99ae9e5d24834082ebf2fe9fd3964fdd80b69ccb"
}
backend/.sqlx/query-ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4.json
Generated
-23
View File
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "SELECT raw_flow->'modules'->$2::int->'retry' FROM queue WHERE id = $1",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "?column?",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": [
"Uuid",
"Int4"
]
},
"nullable": [
null
]
},
"hash": "ae2f005af8ab4b035a907e0c8fc9a9d035f3eb1d9d833041969fce967daa91a4"
}
backend/.sqlx/query-c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb.json
Generated
+29
View File
@@ -0,0 +1,29 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT tag as \"tag!\", COUNT(*) as \"count!\"\n FROM completed_job\n WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)\n GROUP BY tag\n ORDER BY \"count!\" DESC\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "tag!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "count!",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Float8",
"Text"
]
},
"nullable": [
false,
null
]
},
"hash": "c3b5abbf2c9079d597a55f7c63bc83b8b4da98bda204a40f045a62172cfb4ebb"
}
backend/.sqlx/query-d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12.json
Generated
-14
View File
@@ -1,14 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET last_ping = null\n WHERE id = $1",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Uuid"
]
},
"nullable": []
},
"hash": "d0df57fc4cd0be7b541dee081ffbe86b869b01ca1ab10aa17634bb6dae879f12"
}
backend/.sqlx/query-d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43.json
Generated
-23
View File
@@ -1,23 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int4",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Int4",
"Uuid"
]
},
"nullable": [
null
]
},
"hash": "d6c25421bb6513908697ebe74c159da0ef78b5252b6ac669a3c9e545d05c0d43"
}
backend/.sqlx/query-95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4.json → backend/.sqlx/query-e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c.json
Generated
+7 -3
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4 WHERE worker = $5",
"query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,\n occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
"describe": {
"columns": [],
"parameters": {
@@ -9,10 +9,14 @@
"Varchar",
"Int8",
"Int8",
"Text"
"Text",
"Float4",
"Float4",
"Float4",
"Float4"
]
},
"nullable": []
},
"hash": "95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4"
"hash": "e968e879d3c52f7dd502c3cd15fc8fbd983a4a3ab25648c562497a27c74b5c8c"
}
backend/.sqlx/query-f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b.json
Generated
+25
View File
@@ -0,0 +1,25 @@
{
"db_name": "PostgreSQL",
"query": "UPDATE queue\n SET flow_status = JSONB_SET(\n JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),\n ARRAY['modules', $1::TEXT, 'iterator', 'index'],\n ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb\n ),\n last_ping = NULL\n WHERE id = $2\n RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "int4",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Int4",
"Uuid",
"Text",
"Jsonb"
]
},
"nullable": [
null
]
},
"hash": "f916ec232837ece9323675e5f5142e7285f4266a1471e5ffdefadf421a67e44b"
}
backend/Cargo.lock
Generated
+146 -143
View File
File diff suppressed because it is too large Load Diff
backend/Cargo.toml
+3 -3
View File
@@ -1,6 +1,6 @@
[package]
name = "windmill"
version = "1.401.0"
version = "1.402.3"
authors.workspace = true
edition.workspace = true
@@ -27,7 +27,7 @@ members = [
]
[workspace.package]
version = "1.401.0"
version = "1.402.3"
authors = ["Ruben Fiszel <ruben@windmill.dev>"]
edition = "2021"
@@ -44,7 +44,7 @@ default = []
enterprise = ["windmill-worker/enterprise", "windmill-queue/enterprise", "windmill-api/enterprise", "windmill-git-sync/enterprise", "windmill-common/prometheus", "windmill-common/enterprise", "windmill-indexer/enterprise"]
enterprise_saml = ["windmill-api/enterprise_saml"]
stripe = ["windmill-api/stripe"]
benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark"]
benchmark = ["windmill-api/benchmark", "windmill-worker/benchmark", "windmill-queue/benchmark", "windmill-common/benchmark"]
flamegraph = ["windmill-common/flamegraph", "windmill-worker/flamegraph"]
loki = ["windmill-common/loki"]
pg_embed = ["dep:pg-embed"]
backend/migrations/20240924213010_add_last_occupancy_rates.down.sql
+5
View File
@@ -0,0 +1,5 @@
-- Add down migration script here
ALTER TABLE worker_ping
DROP COLUMN occupancy_rate_15s,
DROP COLUMN occupancy_rate_5m,
DROP COLUMN occupancy_rate_30m;
backend/migrations/20240924213010_add_last_occupancy_rates.up.sql
+5
View File
@@ -0,0 +1,5 @@
-- Add up migration script here
ALTER TABLE worker_ping
ADD COLUMN occupancy_rate_15s REAL,
ADD COLUMN occupancy_rate_5m REAL,
ADD COLUMN occupancy_rate_30m REAL;
backend/plot.py
+89
View File
@@ -0,0 +1,89 @@
import json
import matplotlib.pyplot as plt
# Function to load JSON data from a file
def load_json_data(filepath):
with open(filepath, 'r') as file:
data = json.load(file)
return data
# Function to plot two arrays of subarrays with tuples (step_name, duration)
def plot_two_arrays_of_subarrays(arrays1, arrays2):
# Function to calculate sum of durations for each step
def calculate_sums(arrays):
steps = [step for step, _ in arrays[0]['timings']] # Extract steps from the first iteration
sums = {step: 0.0 for step in steps} # Initialize sums dictionary with step names
# Sum up the durations for each step across all subarrays
for subarray in arrays:
for step_name, duration in subarray['timings']:
if step_name not in sums:
sums[step_name] = 0
sums[step_name] += duration
for step_name, duration in sums.items():
sums[step_name] = duration / 1000000000
# Convert the sums dictionary to two lists (for plotting)
step_names = list(sums.keys())
durations = list(sums.values())
return step_names, durations
# Calculate sums for both arrays of subarrays
step_names1, sums1 = calculate_sums(arrays1)
step_names2, sums2 = calculate_sums(arrays2)
# Create two subplots, one on top of the other
fig, (ax1, ax2) = plt.subplots(2, 1, figsize=(10, 12))
# First plot (top) for the first array of subarrays
ax1.bar(step_names1, sums1, color='b')
ax1.set_title('Total Duration per Step - Main Loop')
ax1.set_xlabel('Step Name')
ax1.set_ylabel('Total Duration (s)')
ax1.grid(True, axis='y')
ax1.tick_params(axis='x', rotation=45)
# Second plot (bottom) for the second array of subarrays
ax2.bar(step_names2, sums2, color='r')
ax2.set_title('Total Duration per Step - Result Processor')
ax2.set_xlabel('Step Name')
ax2.set_ylabel('Total Duration (s)')
ax2.grid(True, axis='y')
ax2.tick_params(axis='x', rotation=45)
# Adjust layout so the plots don't overlap
plt.tight_layout()
# Display the plot
plt.show()
# Load arrays from the JSON files
main = load_json_data('/tmp/windmill/profiling_main.json')
result_processor = load_json_data('/tmp/windmill/profiling_result_processor.json')
arrays1 = main['timings']
arrays2 = result_processor['timings']
total_duration1 = main['total_duration']/1000
total_duration2 = result_processor['total_duration']/1000
print(f"Total duration for main: {total_duration1}s")
print(f"Total duration for result processor: {total_duration2}s")
iterations_total = sum(main['iter_durations']) / 1000000000
iterations_total2 = sum(result_processor['iter_durations']) / 1000000000
print(f"Number of iterations: {len(main['iter_durations'])}")
print(f"Total iterations for main: {iterations_total}s")
print(f"Total iterations for result processor: {iterations_total2}s")
# Calculate RPS
rps1 = len(main['iter_durations']) / total_duration1
rps2 = len(result_processor['iter_durations']) / total_duration2
print(f"RPS for main: {rps1}")
print(f"RPS for result processor: {rps2}")
# Plot the data
plot_two_arrays_of_subarrays(arrays1, arrays2)
backend/src/main.rs
+2
View File
@@ -158,6 +158,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
"global",
"global",
"",
&mut None,
)
.await?;
tokio::fs::remove_dir_all(job_dir).await?;
@@ -178,6 +179,7 @@ async fn cache_hub_scripts(file_path: Option<String>) -> anyhow::Result<()> {
"cache_init",
windmill_worker::get_common_bun_proc_envs(None).await,
false,
&mut None,
)
.await?;
} else {
backend/src/monitor.rs
+44 -47
View File
@@ -525,7 +525,7 @@ fn read_log_counters(ts_str: String) -> (usize, usize) {
ok_lines = counter.non_error_count;
err_lines = counter.error_count;
} else {
println!("no counter found for {ts_str}");
// println!("no counter found for {ts_str}");
}
} else {
println!("Error reading log counters 2");
@@ -1071,73 +1071,68 @@ pub async fn monitor_db(
}
pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
let tx = db.begin().await;
if let Ok(mut tx) = tx {
let last_check = sqlx::query_scalar!(
let last_check = sqlx::query_scalar!(
"SELECT created_at FROM metrics WHERE id LIKE 'queue_count_%' ORDER BY created_at DESC LIMIT 1"
)
.fetch_optional(db)
.await
.unwrap_or(Some(chrono::Utc::now()));
let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed);
let save_metrics = last_check
.map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25))
.unwrap_or(true);
let metrics_enabled = METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed);
let save_metrics = last_check
.map(|last_check| chrono::Utc::now() - last_check > chrono::Duration::seconds(25))
.unwrap_or(true);
if metrics_enabled || save_metrics {
let queue_counts = sqlx::query!(
"SELECT tag, count(*) as count FROM queue WHERE
if metrics_enabled || save_metrics {
let queue_counts = sqlx::query!(
"SELECT tag, count(*) as count FROM queue WHERE
scheduled_for <= now() - ('3 seconds')::interval AND running = false
GROUP BY tag"
)
.fetch_all(&mut *tx)
.await
.ok()
.unwrap_or_else(|| vec![]);
)
.fetch_all(db)
.await
.ok()
.unwrap_or_else(|| vec![]);
for q in queue_counts {
let count = q.count.unwrap_or(0);
let tag = q.tag;
if metrics_enabled {
let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
metric.set(count as i64);
}
for q in queue_counts {
let count = q.count.unwrap_or(0);
let tag = q.tag;
if metrics_enabled {
let metric = (*QUEUE_COUNT).with_label_values(&[&tag]);
metric.set(count as i64);
}
// save queue_count and delay metrics per tag
if save_metrics {
// save queue_count and delay metrics per tag
if save_metrics {
sqlx::query!(
"INSERT INTO metrics (id, value) VALUES ($1, $2)",
format!("queue_count_{}", tag),
serde_json::json!(count)
)
.execute(db)
.await
.ok();
if count > 0 {
sqlx::query!(
"INSERT INTO metrics (id, value) VALUES ($1, $2)",
format!("queue_count_{}", tag),
serde_json::json!(count)
)
.execute(&mut *tx)
.await
.ok();
if count > 0 {
sqlx::query!(
"INSERT INTO metrics (id, value)
VALUES ($1, to_jsonb((SELECT EXTRACT(EPOCH FROM now() - scheduled_for)
FROM queue WHERE tag = $2 AND running = false AND scheduled_for <= now() - ('3 seconds')::interval
ORDER BY priority DESC NULLS LAST, scheduled_for, created_at LIMIT 1)))",
ORDER BY priority DESC NULLS LAST, scheduled_for LIMIT 1)))",
format!("queue_delay_{}", tag),
tag
).execute(&mut *tx).await.ok();
}
).execute(db).await.ok();
}
}
}
// clean queue metrics older than 14 days
sqlx::query!(
"DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'"
)
.execute(&mut *tx)
.await
.ok();
tx.commit().await.ok();
}
// clean queue metrics older than 14 days
sqlx::query!(
"DELETE FROM metrics WHERE id LIKE 'queue_%' AND created_at < NOW() - INTERVAL '14 day'"
)
.execute(db)
.await
.ok();
}
pub async fn reload_smtp_config(db: &Pool<Postgres>) {
@@ -1373,6 +1368,8 @@ async fn handle_zombie_jobs<R: rsmq_async::RsmqConnection + Send + Sync + Clone>
rsmq.clone(),
worker_name,
send_result_never_used,
#[cfg(feature = "benchmark")]
&mut windmill_common::bench::BenchmarkIter::new(),
)
.await;
}
backend/windmill-api/openapi.yaml
+47 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.401.0
version: 1.402.3
title: Windmill API
contact:
@@ -5964,6 +5964,43 @@ paths:
schema:
type: integer
/jobs/completed/count_by_tag:
get:
summary: Count jobs by tag
operationId: countJobsByTag
tags:
- job
parameters:
- name: horizon_secs
in: query
description: Past Time horizon in seconds (when to start the count = now - horizon) (default is 3600)
required: false
schema:
type: integer
- name: workspace_id
in: query
description: Specific workspace ID to filter results (optional)
required: false
schema:
type: string
responses:
"200":
description: Job counts by tag
content:
application/json:
schema:
type: array
items:
type: object
properties:
tag:
type: string
count:
type: integer
required:
- tag
- count
/w/{workspace}/jobs_u/get/{id}:
get:
summary: get job
@@ -6498,6 +6535,7 @@ paths:
schema:
type: string
/w/{workspace}/jobs_u/cancel/{id}/{resume_id}/{signature}:
get:
summary: cancel a job for a suspended flow
@@ -9949,6 +9987,8 @@ components:
type: number
aggregate_wait_time_ms:
type: number
suspend:
type: number
required:
- id
- running
@@ -11158,6 +11198,12 @@ components:
type: string
occupancy_rate:
type: number
occupancy_rate_15s:
type: number
occupancy_rate_5m:
type: number
occupancy_rate_30m:
type: number
memory:
type: number
vcpus:
backend/windmill-api/src/configs.rs
+13 -4
View File
@@ -33,7 +33,7 @@ pub fn global_service() -> Router {
#[derive(Serialize, Deserialize, FromRow)]
struct Config {
name: String,
name: Option<String>,
config: serde_json::Value,
}
@@ -41,9 +41,18 @@ async fn list_worker_groups(
authed: ApiAuthed,
Extension(db): Extension<DB>,
) -> error::JsonResult<Vec<Config>> {
let configs_raw = sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
.fetch_all(&db)
.await?;
let mut configs_raw =
sqlx::query_as!(Config, "SELECT * FROM config WHERE name LIKE 'worker__%'")
.fetch_all(&db)
.await?;
// Remove the 'worker__' prefix from all config names
for config in configs_raw.iter_mut() {
if let Some(name) = &config.name {
if name.starts_with("worker__") {
config.name = Some(name.strip_prefix("worker__").unwrap().to_string());
}
}
}
let configs = if !authed.is_admin {
let mut obfuscated_configs: Vec<Config> = vec![];
for config in configs_raw {
backend/windmill-api/src/flows.rs
+1
View File
@@ -242,6 +242,7 @@ pub async fn get_hub_flow_by_id(
#[derive(Deserialize)]
pub struct ToggleWorkspaceErrorHandler {
#[cfg(feature = "enterprise")]
pub muted: Option<bool>,
}
backend/windmill-api/src/jobs.rs
+59 -4
View File
@@ -11,6 +11,7 @@ use axum::http::HeaderValue;
use quick_cache::sync::Cache;
use serde_json::value::RawValue;
use sqlx::Pool;
use windmill_common::error::JsonResult;
use std::collections::HashMap;
#[cfg(feature = "prometheus")]
use std::sync::atomic::Ordering;
@@ -69,7 +70,7 @@ use windmill_common::{
oauth2::HmacSha256,
scripts::{ScriptHash, ScriptLang},
users::username_to_permissioned_as,
utils::{not_found_if_none, now_from_db, paginate, require_admin, Pagination, StripPath},
utils::{not_found_if_none, now_from_db, paginate, paginate_without_limits, require_admin, Pagination, StripPath},
};
#[cfg(all(feature = "enterprise", feature = "parquet"))]
@@ -247,7 +248,7 @@ pub fn workspaced_service() -> Router {
.route("/run/flow_dependencies", post(run_flow_dependencies_job))
}
pub fn global_service() -> Router {
pub fn workspace_unauthed_service() -> Router {
Router::new()
.route(
"/resume/:job_id/:resume_id/:secret",
@@ -291,7 +292,12 @@ pub fn global_service() -> Router {
}
pub fn global_root_service() -> Router {
Router::new().route("/db_clock", get(get_db_clock))
Router::new()
.route("/db_clock", get(get_db_clock))
.route(
"/completed/count_by_tag",
get(count_by_tag),
)
}
#[derive(Deserialize)]
@@ -1248,13 +1254,16 @@ pub fn list_queue_jobs_query(
w_id: &str,
lq: &ListQueueQuery,
fields: &[&str],
pagination: Pagination,
join_outstanding_wait_times: bool,
tags: Option<Vec<&str>>,
) -> SqlBuilder {
let (limit, offset) = paginate_without_limits(pagination);
let mut sqlb = SqlBuilder::select_from("queue")
.fields(fields)
.order_by("created_at", lq.order_desc.unwrap_or(true))
.limit(1000)
.limit(limit)
.offset(offset)
.clone();
if let Some(tags) = tags {
@@ -1267,6 +1276,7 @@ pub fn list_queue_jobs_query(
#[derive(Serialize, FromRow)]
struct ListableQueuedJob {
pub id: Uuid,
pub running: bool,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub started_at: Option<chrono::DateTime<chrono::Utc>>,
@@ -1289,6 +1299,7 @@ async fn list_queue_jobs(
authed: ApiAuthed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
Query(lq): Query<ListQueueQuery>,
) -> error::JsonResult<Vec<ListableQueuedJob>> {
let sql = list_queue_jobs_query(
@@ -1296,6 +1307,7 @@ async fn list_queue_jobs(
&lq,
&[
"id",
"running",
"created_by",
"created_at",
"started_at",
@@ -1315,6 +1327,7 @@ async fn list_queue_jobs(
"priority",
"workspace_id",
],
pagination,
false,
get_scope_tags(&authed),
)
@@ -1570,6 +1583,7 @@ async fn list_jobs(
) -> error::JsonResult<Vec<Job>> {
check_scopes(&authed, || format!("jobs:listjobs"))?;
let limit = pagination.per_page.unwrap_or(1000);
let (per_page, offset) = paginate(pagination);
let lqc = lq.clone();
@@ -1601,6 +1615,7 @@ async fn list_jobs(
&w_id,
&ListQueueQuery { order_desc: Some(true), ..lq.into() },
UnifiedJob::queued_job_fields(),
Pagination { per_page: Some(limit), page: None },
true,
get_scope_tags(&authed),
);
@@ -5100,6 +5115,46 @@ async fn get_completed_job_result(
Ok(Json(result).into_response())
}
#[derive(Deserialize)]
struct CountByTagQuery {
horizon_secs: Option<i64>,
workspace_id: Option<String>,
}
#[derive(Serialize)]
struct TagCount {
tag: String,
count: i64,
}
async fn count_by_tag(
ApiAuthed { email, ..}: ApiAuthed,
Extension(db): Extension<DB>,
Query(query): Query<CountByTagQuery>,
) -> JsonResult<Vec<TagCount>> {
require_super_admin(&db, &email).await?;
let horizon = query.horizon_secs.unwrap_or(3600); // Default to 1 hour if not specified
let counts = sqlx::query_as!(
TagCount,
r#"
SELECT tag as "tag!", COUNT(*) as "count!"
FROM completed_job
WHERE started_at > NOW() - make_interval(secs => $1) AND ($2::text IS NULL OR workspace_id = $2)
GROUP BY tag
ORDER BY "count!" DESC
"#,
horizon as f64,
query.workspace_id
)
.fetch_all(&db)
.await?;
Ok(Json(counts))
}
#[derive(Serialize)]
struct CompletedJobResult {
started: Option<bool>,
backend/windmill-api/src/lib.rs
+1 -1
View File
@@ -330,7 +330,7 @@ pub async fn run_server(
)
.nest(
"/w/:workspace_id/jobs_u",
jobs::global_service().layer(cors.clone()),
jobs::workspace_unauthed_service().layer(cors.clone()),
)
.nest(
"/w/:workspace_id/resources_u",
backend/windmill-api/src/scripts.rs
+1
View File
@@ -967,6 +967,7 @@ async fn list_paths(
#[derive(Deserialize)]
pub struct ToggleWorkspaceErrorHandler {
#[cfg(feature = "enterprise")]
pub muted: Option<bool>,
}
backend/windmill-api/src/workers.rs
+9 -1
View File
@@ -54,6 +54,12 @@ struct WorkerPing {
#[serde(skip_serializing_if = "Option::is_none")]
occupancy_rate: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
occupancy_rate_15s: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
occupancy_rate_5m: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
occupancy_rate_30m: Option<f32>,
#[serde(skip_serializing_if = "Option::is_none")]
memory: Option<i64>,
#[serde(skip_serializing_if = "Option::is_none")]
vcpus: Option<i64>,
@@ -88,7 +94,9 @@ async fn list_worker_pings(
let rows = sqlx::query_as!(
WorkerPing,
"SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage
"SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed,
CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as last_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as last_job_workspace_id,
custom_tags, worker_group, wm_version, occupancy_rate, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, memory, vcpus, memory_usage, wm_memory_usage
FROM worker_ping
WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)
ORDER BY ping_at desc LIMIT $2 OFFSET $3",
backend/windmill-api/src/workspaces.rs
+7 -2
View File
@@ -42,7 +42,10 @@ use windmill_common::schedule::Schedule;
use windmill_common::users::username_to_permissioned_as;
use windmill_common::variables::build_crypt;
use windmill_common::worker::{to_raw_value, CLOUD_HOSTED};
use windmill_common::workspaces::{WorkspaceDeploymentUISettings, WorkspaceGitSyncSettings};
#[cfg(feature = "enterprise")]
use windmill_common::workspaces::WorkspaceDeploymentUISettings;
#[cfg(feature = "enterprise")]
use windmill_common::workspaces::WorkspaceGitSyncSettings;
use windmill_common::{
error::{to_anyhow, Error, JsonResult, Result},
flows::Flow,
@@ -991,6 +994,7 @@ async fn edit_large_file_storage_config(
#[derive(Deserialize)]
pub struct EditGitSyncConfig {
#[cfg(feature = "enterprise")]
pub git_sync_settings: Option<WorkspaceGitSyncSettings>,
}
@@ -1056,6 +1060,7 @@ async fn edit_git_sync_config(
#[derive(Deserialize)]
struct EditDeployUIConfig {
#[cfg(feature = "enterprise")]
deploy_ui_settings: Option<WorkspaceDeploymentUISettings>,
}
@@ -1064,7 +1069,6 @@ async fn edit_deploy_ui_config(
_authed: ApiAuthed,
Extension(_db): Extension<DB>,
Path(_w_id): Path<String>,
Json(_new_config): Json<EditDeployUIConfig>,
) -> Result<String> {
return Err(Error::BadRequest(
"Deployment UI is only available on Windmill Enterprise Edition".to_string(),
@@ -1122,6 +1126,7 @@ async fn edit_deploy_ui_config(
#[derive(Deserialize)]
pub struct EditDefaultApp {
#[cfg(feature = "enterprise")]
pub default_app_path: Option<String>,
}
backend/windmill-common/Cargo.toml
+1
View File
@@ -11,6 +11,7 @@ jemalloc = ["dep:tikv-jemalloc-ctl"]
prometheus = ["dep:prometheus"]
flamegraph = ["dep:tracing-flame"]
loki = ["dep:tracing-loki"]
benchmark = []
parquet = ["dep:object_store", "dep:aws-config", "dep:aws-sdk-sts"]
[lib]
backend/windmill-common/src/bench.rs
+213
View File
@@ -0,0 +1,213 @@
use crate::{
worker::{write_file, TMP_DIR},
DB,
};
use serde::Serialize;
use tokio::time::Instant;
#[derive(Serialize)]
pub struct BenchmarkInfo {
#[serde(skip)]
pub start: Instant,
#[serde(skip)]
pub iters: u64,
timings: Vec<BenchmarkIter>,
pub iter_durations: Vec<u64>,
pub total_duration: Option<u64>,
}
impl BenchmarkInfo {
pub fn new() -> Self {
BenchmarkInfo {
iters: 0,
timings: vec![],
start: Instant::now(),
iter_durations: vec![],
total_duration: None,
}
}
pub fn add_iter(&mut self, bench: BenchmarkIter, inc_iters: bool) {
if inc_iters {
self.iters += 1;
}
let elapsed_total = bench.start.elapsed().as_nanos() as u64;
self.timings.push(bench);
self.iter_durations.push(elapsed_total);
}
pub fn write_to_file(&mut self, path: &str) -> anyhow::Result<()> {
let total_duration = self.start.elapsed().as_millis() as u64;
self.total_duration = Some(total_duration as u64);
println!(
"Writing benchmark {path}, duration of benchmark: {total_duration}s and RPS: {}",
self.iters as f64 / total_duration as f64
);
write_file(TMP_DIR, path, &serde_json::to_string(&self).unwrap()).expect("write profiling");
Ok(())
}
}
#[derive(Serialize)]
pub struct BenchmarkIter {
#[serde(skip)]
pub start: Instant,
#[serde(skip)]
last_instant: Instant,
last_step: String,
timings: Vec<(String, u32)>,
}
impl BenchmarkIter {
pub fn new() -> Self {
BenchmarkIter {
last_instant: Instant::now(),
timings: vec![],
start: Instant::now(),
last_step: String::new(),
}
}
pub fn add_timing(&mut self, name: &str) {
let elapsed = self.last_instant.elapsed().as_nanos() as u32;
self.timings
.push((format!("{}->{}", self.last_step, name), elapsed));
self.last_instant = Instant::now();
self.last_step = name.to_string();
}
}
pub async fn benchmark_init(benchmark_jobs: i32, db: &DB) {
use crate::{jobs::JobKind, scripts::ScriptLang};
let benchmark_kind = std::env::var("BENCHMARK_KIND").unwrap_or("noop".to_string());
if benchmark_jobs > 0 {
match benchmark_kind.as_str() {
"dedicated" => {
// you need to create the script first, check https://github.com/windmill-labs/windmill/blob/b76a92cfe454c686f005c65f534e29e039f3c706/benchmarks/lib.ts#L47
let hash = sqlx::query_scalar!(
"SELECT hash FROM script WHERE path = $1 AND workspace_id = $2",
"f/benchmarks/dedicated",
"admins"
)
.fetch_one(db)
.await
.unwrap_or_else(|_e| panic!("failed to insert dedicated jobs"));
sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))",
hash,
"f/benchmarks/dedicated",
JobKind::Script as JobKind,
ScriptLang::Bun as ScriptLang,
"admins:f/benchmarks/dedicated",
"admin",
"u/admin",
"admin@windmill.dev",
chrono::Utc::now(),
"admins",
benchmark_jobs
)
.execute(db)
.await.unwrap_or_else(|_e| panic!("failed to insert dedicated jobs"));
}
"parallelflow" => {
//create dedicated script
sqlx::query!("INSERT INTO script (summary, description, dedicated_worker, content, workspace_id, path, hash, language, tag, created_by, lock) VALUES ('', '', true, $1, $2, $3, $4, $5, $6, $7, '') ON CONFLICT (workspace_id, hash) DO NOTHING",
"export async function main() {
console.log('hello world');
}",
"admins",
"u/admin/parallelflow",
1234567890,
ScriptLang::Deno as ScriptLang,
"flow",
"admin",
)
.execute(db)
.await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs {_e:#}"));
sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id, raw_flow, flow_status) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12 FROM generate_series(1, 1))",
None::<i64>,
None::<String>,
JobKind::FlowPreview as JobKind,
ScriptLang::Deno as ScriptLang,
"flow",
"admin",
"u/admin",
"admin@windmill.dev",
chrono::Utc::now(),
"admins",
serde_json::from_str::<serde_json::Value>(r#"
{
"modules": [
{
"id": "a",
"value": {
"type": "forloopflow",
"modules": [
{
"id": "b",
"value": {
"path": "u/admin/parallelflow",
"type": "script",
"tag_override": "",
"input_transforms": {}
},
"summary": "calctest"
}
],
"iterator": {
"expr": "[...new Array(300)]",
"type": "javascript"
},
"parallel": true,
"parallelism": 10,
"skip_failures": true
}
}
],
"preprocessor_module": null
}
"#).unwrap(),
serde_json::from_str::<serde_json::Value>(r#"
{
"step": 0,
"modules": [
{
"id": "a",
"type": "WaitingForPriorSteps"
}
],
"cleanup_module": {},
"failure_module": {
"id": "failure",
"type": "WaitingForPriorSteps"
},
"preprocessor_module": null
}
"#).unwrap()
)
.execute(db)
.await.unwrap_or_else(|_e| panic!("failed to insert parallelflow jobs"));
}
_ => {
sqlx::query!("INSERT INTO queue (id, script_hash, script_path, job_kind, language, tag, created_by, permissioned_as, email, scheduled_for, workspace_id) (SELECT gen_random_uuid(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10 FROM generate_series(1, $11))",
None::<i64>,
None::<String>,
JobKind::Noop as JobKind,
ScriptLang::Deno as ScriptLang,
"deno",
"admin",
"u/admin",
"admin@windmill.dev",
chrono::Utc::now(),
"admins",
benchmark_jobs
)
.execute(db)
.await.unwrap_or_else(|_e| panic!("failed to insert noop jobs"));
}
}
}
}
backend/windmill-common/src/ee.rs~main
+73
View File
@@ -0,0 +1,73 @@
#[cfg(feature = "enterprise")]
use crate::db::DB;
use crate::ee::LicensePlan::Community;
#[cfg(feature = "enterprise")]
use crate::error;
use serde::Deserialize;
use std::sync::Arc;
use tokio::sync::RwLock;
lazy_static::lazy_static! {
pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
}
pub enum LicensePlan {
Community,
Pro,
Enterprise,
}
pub async fn get_license_plan() -> LicensePlan {
// Implementation is not open source
return Community;
}
#[derive(Deserialize)]
#[serde(untagged)]
pub enum CriticalErrorChannel {}
pub enum CriticalAlertKind {
#[cfg(feature = "enterprise")]
CriticalError,
#[cfg(feature = "enterprise")]
RecoveredCriticalError,
}
#[cfg(feature = "enterprise")]
pub async fn send_critical_alert(
_error_message: String,
_db: &DB,
_kind: CriticalAlertKind,
_channels: Option<Vec<CriticalErrorChannel>>,
) {
}
#[cfg(feature = "enterprise")]
pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
// Implementation is not open source
}
#[cfg(feature = "enterprise")]
pub async fn renew_license_key(
_http_client: &reqwest::Client,
_db: &crate::db::DB,
_key: Option<String>,
_manual: bool,
) -> String {
// Implementation is not open source
"".to_string()
}
#[cfg(feature = "enterprise")]
pub async fn create_customer_portal_session(
_http_client: &reqwest::Client,
_key: Option<String>,
) -> error::Result<String> {
// Implementation is not open source
Ok("".to_string())
}
#[cfg(feature = "enterprise")]
pub async fn worker_groups_alerts(_db: &DB) {}
backend/windmill-common/src/ee.rs~main_0
+76
View File
@@ -0,0 +1,76 @@
#[cfg(feature = "enterprise")]
use crate::db::DB;
use crate::ee::LicensePlan::Community;
#[cfg(feature = "enterprise")]
use crate::error;
use serde::Deserialize;
use std::sync::Arc;
use tokio::sync::RwLock;
lazy_static::lazy_static! {
pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
}
pub enum LicensePlan {
Community,
Pro,
Enterprise,
}
pub async fn get_license_plan() -> LicensePlan {
// Implementation is not open source
return Community;
}
#[derive(Deserialize)]
#[serde(untagged)]
pub enum CriticalErrorChannel {
Email { email: String },
Slack { slack_channel: String },
}
pub enum CriticalAlertKind {
#[cfg(feature = "enterprise")]
CriticalError,
#[cfg(feature = "enterprise")]
RecoveredCriticalError,
}
#[cfg(feature = "enterprise")]
pub async fn send_critical_alert(
_error_message: String,
_db: &DB,
_kind: CriticalAlertKind,
_channels: Option<Vec<CriticalErrorChannel>>,
) {
}
#[cfg(feature = "enterprise")]
pub async fn schedule_key_renewal(_http_client: &reqwest::Client, _db: &crate::db::DB) -> () {
// Implementation is not open source
}
#[cfg(feature = "enterprise")]
pub async fn renew_license_key(
_http_client: &reqwest::Client,
_db: &crate::db::DB,
_key: Option<String>,
_manual: bool,
) -> String {
// Implementation is not open source
"".to_string()
}
#[cfg(feature = "enterprise")]
pub async fn create_customer_portal_session(
_http_client: &reqwest::Client,
_key: Option<String>,
) -> error::Result<String> {
// Implementation is not open source
Ok("".to_string())
}
#[cfg(feature = "enterprise")]
pub async fn worker_groups_alerts(_db: &DB) {}
backend/windmill-common/src/flows.rs
+27
View File
@@ -285,6 +285,23 @@ pub struct FlowModuleValueWithParallel {
pub parallelism: Option<u16>,
}
#[derive(Deserialize)]
pub struct FlowModuleValueWithSkipFailures {
pub skip_failures: Option<bool>,
pub parallel: Option<bool>,
pub parallelism: Option<u16>,
}
#[derive(Deserialize)]
pub struct BranchWithSkipFailures {
pub skip_failure: Option<bool>,
}
#[derive(Deserialize)]
pub struct FlowModuleWithBranches {
pub branches: Vec<BranchWithSkipFailures>,
}
impl FlowModule {
pub fn id_append(&mut self, s: &str) {
self.id = format!("{}-{}", self.id, s);
@@ -293,6 +310,16 @@ impl FlowModule {
serde_json::from_str::<FlowModuleValue>(self.value.get()).map_err(crate::error::to_anyhow)
}
pub fn get_value_with_skip_failures(&self) -> anyhow::Result<FlowModuleValueWithSkipFailures> {
serde_json::from_str::<FlowModuleValueWithSkipFailures>(self.value.get())
.map_err(crate::error::to_anyhow)
}
pub fn get_branches_skip_failures(&self) -> anyhow::Result<FlowModuleWithBranches> {
serde_json::from_str::<FlowModuleWithBranches>(self.value.get())
.map_err(crate::error::to_anyhow)
}
pub fn is_flow(&self) -> bool {
self.get_type().is_ok_and(|x| x == "flow")
}
backend/windmill-common/src/lib.rs
+13
View File
@@ -17,6 +17,8 @@ use scripts::ScriptLang;
use sqlx::{Pool, Postgres};
pub mod apps;
#[cfg(feature = "benchmark")]
pub mod bench;
pub mod db;
pub mod ee;
pub mod error;
@@ -51,6 +53,17 @@ pub const DEFAULT_MAX_CONNECTIONS_INDEXER: u32 = 5;
pub const DEFAULT_HUB_BASE_URL: &str = "https://hub.windmill.dev";
#[macro_export]
macro_rules! add_time {
($bench:expr, $name:expr) => {
#[cfg(feature = "benchmark")]
{
$bench.add_timing($name);
// println!("{}: {:?}", $z, $y.elapsed());
}
};
}
lazy_static::lazy_static! {
pub static ref METRICS_PORT: u16 = std::env::var("METRICS_PORT")
.ok()
backend/windmill-common/src/utils.rs
+7 -1
View File
@@ -38,7 +38,7 @@ lazy_static::lazy_static! {
.build().unwrap();
}
#[derive(Deserialize)]
#[derive(Deserialize, Clone)]
pub struct Pagination {
pub page: Option<usize>,
pub per_page: Option<usize>,
@@ -82,6 +82,12 @@ pub fn paginate(pagination: Pagination) -> (usize, usize) {
(per_page, offset)
}
pub fn paginate_without_limits(pagination: Pagination) -> (usize, usize) {
let per_page = pagination.per_page.unwrap_or(MAX_PER_PAGE);
let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page;
(per_page, offset)
}
pub async fn now_from_db<'c, E: sqlx::PgExecutor<'c>>(
db: E,
) -> Result<chrono::DateTime<chrono::Utc>> {
backend/windmill-queue/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ path = "src/lib.rs"
default = []
enterprise = ["windmill-common/enterprise"]
cloud = []
benchmark = []
benchmark = ["windmill-common/benchmark"]
prometheus = ["dep:prometheus"]
[dependencies]
backend/windmill-queue/src/jobs.rs
+44 -21
View File
@@ -37,9 +37,9 @@ use ulid::Ulid;
use uuid::Uuid;
use windmill_audit::audit_ee::{audit_log, AuditAuthor};
use windmill_audit::ActionKind;
#[cfg(not(feature = "enterprise"))]
use windmill_common::worker::PriorityTags;
use windmill_common::{
add_time,
auth::{fetch_authed_from_permissioned_as, permissioned_as_to_username},
db::{Authed, UserDB},
error::{self, to_anyhow, Error},
@@ -62,9 +62,12 @@ use windmill_common::{
to_raw_value, DEFAULT_TAGS_PER_WORKSPACE, DEFAULT_TAGS_WORKSPACES, NO_LOGS, WORKER_CONFIG,
WORKER_PULL_QUERIES, WORKER_SUSPENDED_PULL_QUERY,
},
BASE_URL, DB, METRICS_ENABLED,
DB, METRICS_ENABLED,
};
#[cfg(feature = "enterprise")]
use windmill_common::BASE_URL;
#[cfg(feature = "cloud")]
use windmill_common::users::SUPERADMIN_SYNC_EMAIL;
@@ -125,10 +128,12 @@ const MAX_FREE_CONCURRENT_RUNS: i32 = 30;
const ERROR_HANDLER_USERNAME: &str = "error_handler";
const SCHEDULE_ERROR_HANDLER_USERNAME: &str = "schedule_error_handler";
#[cfg(feature = "enterprise")]
const SCHEDULE_RECOVERY_HANDLER_USERNAME: &str = "schedule_recovery_handler";
const ERROR_HANDLER_USER_GROUP: &str = "g/error_handler";
const ERROR_HANDLER_USER_EMAIL: &str = "error_handler@windmill.dev";
const SCHEDULE_ERROR_HANDLER_USER_EMAIL: &str = "schedule_error_handler@windmill.dev";
#[cfg(feature = "enterprise")]
const SCHEDULE_RECOVERY_HANDLER_USER_EMAIL: &str = "schedule_recovery_handler@windmill.dev";
#[derive(Clone, Debug)]
@@ -174,6 +179,8 @@ pub async fn cancel_single_job<'c>(
rsmq.clone(),
"server",
false,
#[cfg(feature = "benchmark")]
&mut windmill_common::bench::BenchmarkIter::new(),
)
.await;
@@ -477,6 +484,7 @@ where
#[derive(Deserialize)]
struct RawFlowFailureModule {
#[cfg(feature = "enterprise")]
failure_module: Option<Box<RawValue>>,
}
@@ -490,6 +498,7 @@ pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Sen
rsmq: Option<R>,
_worker_name: &str,
flow_is_done: bool,
#[cfg(feature = "benchmark")] bench: &mut windmill_common::bench::BenchmarkIter,
) -> Result<WrappedError, Error> {
#[cfg(feature = "prometheus")]
register_metric(
@@ -527,6 +536,8 @@ pub async fn add_completed_job_error<R: rsmq_async::RsmqConnection + Clone + Sen
canceled_by,
rsmq,
flow_is_done,
#[cfg(feature = "benchmark")]
bench,
)
.await?;
Ok(result)
@@ -550,10 +561,12 @@ pub async fn add_completed_job<
canceled_by: Option<CanceledBy>,
rsmq: Option<R>,
flow_is_done: bool,
#[cfg(feature = "benchmark")] bench: &mut windmill_common::bench::BenchmarkIter,
) -> Result<Uuid, Error> {
// tracing::error!("Start");
// let start = tokio::time::Instant::now();
add_time!(bench, "add_completed_job start");
if !result.is_valid_json() {
return Err(Error::InternalErr(
"Result of job is invalid json (empty)".to_string(),
@@ -561,6 +574,7 @@ pub async fn add_completed_job<
}
let mut tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
let job_id = queued_job.id;
// tracing::error!("1 {:?}", start.elapsed());
@@ -571,6 +585,7 @@ pub async fn add_completed_job<
);
let mem_peak = mem_peak.max(queued_job.mem_peak.unwrap_or(0));
add_time!(bench, "add_completed_job query START");
let _duration: i64 = sqlx::query_scalar!(
"INSERT INTO completed_job AS cj
( workspace_id
@@ -642,6 +657,8 @@ pub async fn add_completed_job<
.map_err(|e| Error::InternalErr(format!("Could not add completed job {job_id}: {e:#}")))?;
// tracing::error!("2 {:?}", start.elapsed());
add_time!(bench, "add_completed_job query END");
if !queued_job.is_flow_step {
if _duration > 500
&& (queued_job.job_kind == JobKind::Script || queued_job.job_kind == JobKind::Preview)
@@ -671,6 +688,7 @@ pub async fn add_completed_job<
}
}
// tracing::error!("Added completed job {:#?}", queued_job);
#[cfg(feature = "enterprise")]
let mut skip_downstream_error_handlers = false;
tx = delete_job(tx, &queued_job.workspace_id, job_id).await?;
// tracing::error!("3 {:?}", start.elapsed());
@@ -716,7 +734,10 @@ pub async fn add_completed_job<
.await?;
if let Some(schedule) = schedule {
skip_downstream_error_handlers = schedule.ws_error_handler_muted;
#[cfg(feature = "enterprise")]
{
skip_downstream_error_handlers = schedule.ws_error_handler_muted;
}
// script or flow that failed on start and might not have been rescheduled
let schedule_next_tick = !queued_job.is_flow()
@@ -749,6 +770,7 @@ pub async fn add_completed_job<
};
}
#[cfg(feature = "enterprise")]
if let Err(err) = apply_schedule_handlers(
rsmq.clone(),
db,
@@ -1324,6 +1346,8 @@ struct CompletedJobSubset {
result: Option<sqlx::types::Json<Box<RawValue>>>,
started_at: chrono::DateTime<chrono::Utc>,
}
#[cfg(feature = "enterprise")]
async fn apply_schedule_handlers<
'a,
'c,
@@ -1342,7 +1366,6 @@ async fn apply_schedule_handlers<
job_priority: Option<i16>,
) -> windmill_common::error::Result<()> {
if !success {
#[cfg(feature = "enterprise")]
if let Some(on_failure_path) = schedule.on_failure.clone() {
let times = schedule.on_failure_times.unwrap_or(1).max(1);
let exact = schedule.on_failure_exact.unwrap_or(false);
@@ -1392,7 +1415,6 @@ async fn apply_schedule_handlers<
.await?;
}
} else {
#[cfg(feature = "enterprise")]
if let Some(ref on_success_path) = schedule.on_success {
handle_successful_schedule(
db,
@@ -1410,7 +1432,6 @@ async fn apply_schedule_handlers<
.await?;
}
#[cfg(feature = "enterprise")]
if let Some(ref on_recovery_path) = schedule.on_recovery.clone() {
let tx: QueueTransaction<'_, R> = (rsmq.clone(), db.begin().await?).into();
let times = schedule.on_recovery_times.unwrap_or(1).max(1);
@@ -1579,6 +1600,7 @@ fn sanitize_result<T: Serialize + Send + Sync>(result: Json<&T>) -> HashMap<Stri
// is_flow: boolean,
// extra_args: serde_json::Value
// }
#[cfg(feature = "enterprise")]
async fn handle_recovered_schedule<
'a,
'c,
@@ -1671,6 +1693,7 @@ async fn handle_recovered_schedule<
Ok(())
}
#[cfg(feature = "enterprise")]
async fn handle_successful_schedule<
'a,
'c,
@@ -1755,9 +1778,9 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
db: &Pool<Postgres>,
rsmq: Option<R>,
suspend_first: bool,
) -> windmill_common::error::Result<Option<QueuedJob>> {
) -> windmill_common::error::Result<(Option<QueuedJob>, bool)> {
loop {
let job = pull_single_job_and_mark_as_running_no_concurrency_limit(
let (job, suspended) = pull_single_job_and_mark_as_running_no_concurrency_limit(
db,
rsmq.clone(),
suspend_first,
@@ -1765,7 +1788,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
.await?;
if job.is_none() {
return Ok(None);
return Ok((None, suspended));
}
let has_concurent_limit = job.as_ref().unwrap().concurrent_limit.is_some();
@@ -1785,7 +1808,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
if METRICS_ENABLED.load(std::sync::atomic::Ordering::Relaxed) {
QUEUE_PULL_COUNT.inc();
}
return Ok(Option::Some(pulled_job));
return Ok((Option::Some(pulled_job), suspended));
}
let itx = db.begin().await?;
@@ -1886,7 +1909,7 @@ pub async fn pull<R: rsmq_async::RsmqConnection + Send + Clone>(
QUEUE_PULL_COUNT.inc();
}
tx.commit().await?;
return Ok(Option::Some(pulled_job));
return Ok((Option::Some(pulled_job), suspended));
}
let x = sqlx::query_scalar!(
"UPDATE concurrency_counter SET job_uuids = job_uuids - $2 WHERE concurrency_id = $1 RETURNING (SELECT COUNT(*) FROM jsonb_object_keys(job_uuids))",
@@ -2013,8 +2036,8 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
db: &Pool<Postgres>,
rsmq: Option<R>,
suspend_first: bool,
) -> windmill_common::error::Result<Option<QueuedJob>> {
let job: Option<QueuedJob> = if let Some(mut rsmq) = rsmq {
) -> windmill_common::error::Result<(Option<QueuedJob>, bool)> {
let job_and_suspended: (Option<QueuedJob>, bool) = if let Some(mut rsmq) = rsmq {
#[cfg(feature = "benchmark")]
let instant = Instant::now();
@@ -2072,9 +2095,9 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
#[cfg(feature = "benchmark")]
println!("rsmq 2: {:?}", instant.elapsed());
m2r
(m2r, false)
} else {
None
(None, false)
}
} else {
/* Jobs can be started if they:
@@ -2088,7 +2111,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
if query.is_empty() {
tracing::warn!("No suspended pull queries available");
return Ok(None);
return Ok((None, false));
}
let r = if suspend_first {
@@ -2108,7 +2131,7 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
if queries.is_empty() {
tracing::warn!("No pull queries available");
return Ok(None);
return Ok((None, false));
}
for query in queries.iter() {
@@ -2126,12 +2149,12 @@ async fn pull_single_job_and_mark_as_running_no_concurrency_limit<
// #[cfg(feature = "benchmark")]
// println!("pull query: {:?}", instant.elapsed());
highest_priority_job
(highest_priority_job, false)
} else {
r
(r, true)
}
};
Ok(job)
Ok(job_and_suspended)
}
pub async fn custom_concurrency_key(
backend/windmill-worker/Cargo.toml
+1 -1
View File
@@ -12,7 +12,7 @@ path = "src/lib.rs"
default = []
prometheus = ["dep:prometheus", "windmill-common/prometheus"]
enterprise = ["windmill-queue/enterprise", "windmill-git-sync/enterprise", "windmill-common/enterprise", "dep:gcp_auth", "dep:pem", "dep:tiberius", "dep:tokio-util", "dep:openidconnect"]
benchmark = ["windmill-queue/benchmark"]
benchmark = ["windmill-queue/benchmark", "windmill-common/benchmark"]
flamegraph = []
parquet = ["windmill-common/parquet", "dep:object_store"]
flow_testing = []
backend/windmill-worker/nsjail/download.py.config.proto
+6
View File
@@ -80,6 +80,12 @@ mount {
rw: true
}
mount {
src: "/dev/urandom"
dst: "/dev/urandom"
is_bind: true
}
exec_bin {
path: "/bin/sh"
arg: "/download_deps.sh"
backend/windmill-worker/nsjail/download_deps.py.sh
+1 -1
View File
@@ -19,6 +19,6 @@ then
echo "\$TRUSTED_HOST is set to $TRUSTED_HOST"
fi
CMD="/usr/local/bin/python3 -m pip install -v $REQ -I -t $TARGET --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
CMD="/usr/local/bin/python3 -m pip install -v \"$REQ\" -I -t \"$TARGET\" --no-cache --no-color --no-deps --isolated --no-warn-conflicts --disable-pip-version-check $INDEX_URL_ARG $EXTRA_INDEX_URL_ARG $TRUSTED_HOST_ARG"
echo $CMD
eval $CMD
backend/windmill-worker/nsjail/run.bun.config.proto
+1
View File
@@ -120,6 +120,7 @@ mount {
src: "{JOB_DIR}/args.json"
dst: "/tmp/{LANG}/args.json"
is_bind: true
rw: true
}
mount {
backend/windmill-worker/nsjail/run.python3.config.proto
+1
View File
@@ -81,6 +81,7 @@ mount {
src: "{JOB_DIR}/args.json"
dst: "/tmp/args.json"
is_bind: true
rw: true
}
mount {
backend/windmill-worker/src/ansible_executor.rs
+12 -2
View File
@@ -21,9 +21,10 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
bash_executor::BIN_BASH,
common::{
get_reserved_variables, handle_child, read_and_check_result, start_child_process,
transform_json,
get_reserved_variables, read_and_check_result, start_child_process, transform_json,
OccupancyMetrics,
},
handle_child::handle_child,
python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile},
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
TZ_ENV,
@@ -50,6 +51,7 @@ async fn handle_ansible_python_deps(
worker_dir: &str,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<Vec<String>> {
create_dependencies_dir(job_dir).await;
@@ -79,6 +81,7 @@ async fn handle_ansible_python_deps(
db,
worker_name,
w_id,
&mut Some(occupancy_metrics),
)
.await
.map_err(|e| {
@@ -102,6 +105,7 @@ async fn handle_ansible_python_deps(
worker_name,
job_dir,
worker_dir,
&mut Some(occupancy_metrics),
)
.await?;
additional_python_paths.append(&mut venv_path);
@@ -118,6 +122,7 @@ async fn install_galaxy_collections(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
db: &sqlx::Pool<sqlx::Postgres>,
occupancy_metrics: &mut OccupancyMetrics,
) -> anyhow::Result<()> {
write_file(job_dir, "requirements.yml", collections_yml)?;
@@ -160,6 +165,7 @@ async fn install_galaxy_collections(
"ansible galaxy install",
None,
false,
&mut Some(occupancy_metrics),
)
.await?;
@@ -198,6 +204,7 @@ pub async fn handle_ansible_job(
shared_mount: &str,
base_internal_url: &str,
envs: HashMap<String, String>,
occupancy_metrics: &mut OccupancyMetrics,
) -> windmill_common::error::Result<Box<RawValue>> {
check_ansible_exists()?;
@@ -216,6 +223,7 @@ pub async fn handle_ansible_job(
worker_dir,
mem_peak,
canceled_by,
occupancy_metrics,
)
.await?;
@@ -289,6 +297,7 @@ pub async fn handle_ansible_job(
mem_peak,
canceled_by,
db,
occupancy_metrics,
)
.await?;
}
@@ -424,6 +433,7 @@ fi
"python run",
job.timeout,
false,
&mut Some(occupancy_metrics),
)
.await?;
read_and_check_result(job_dir).await
backend/windmill-worker/src/bash_executor.rs
+8 -2
View File
@@ -24,9 +24,10 @@ lazy_static::lazy_static! {
use crate::{
common::{
build_args_map, get_reserved_variables, handle_child, read_file, read_file_content,
start_child_process,
build_args_map, get_reserved_variables, read_file, read_file_content, start_child_process,
OccupancyMetrics,
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
POWERSHELL_CACHE_DIR, POWERSHELL_PATH, TZ_ENV,
};
@@ -49,6 +50,7 @@ pub async fn handle_bash_job(
base_internal_url: &str,
worker_name: &str,
envs: HashMap<String, String>,
occupancy_metrics: &mut OccupancyMetrics,
) -> Result<Box<RawValue>, Error> {
let logs1 = "\n\n--- BASH CODE EXECUTION ---\n".to_string();
append_logs(&job.id, &job.workspace_id, logs1, db).await;
@@ -142,6 +144,7 @@ pub async fn handle_bash_job(
"bash run",
job.timeout,
true,
&mut Some(occupancy_metrics),
)
.await?;
@@ -194,6 +197,7 @@ pub async fn handle_powershell_job(
base_internal_url: &str,
worker_name: &str,
envs: HashMap<String, String>,
occupancy_metrics: &mut OccupancyMetrics,
) -> Result<Box<RawValue>, Error> {
let pwsh_args = {
let args = build_args_map(job, client, db).await?.map(Json);
@@ -275,6 +279,7 @@ pub async fn handle_powershell_job(
"powershell install",
job.timeout,
false,
&mut Some(occupancy_metrics),
)
.await?;
}
@@ -378,6 +383,7 @@ $env:PSModulePath = \"{}:$PSModulePathBackup\"",
"powershell run",
job.timeout,
false,
&mut Some(occupancy_metrics),
)
.await?;
backend/windmill-worker/src/bigquery_executor.rs
+4 -1
View File
@@ -14,7 +14,8 @@ use windmill_queue::{CanceledBy, HTTP_CLIENT};
use serde::Deserialize;
use crate::common::run_future_with_polling_update_job_poller;
use crate::common::OccupancyMetrics;
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::{
common::{build_args_values, resolve_job_timeout},
AuthedClientBackgroundTask,
@@ -210,6 +211,7 @@ pub async fn do_bigquery(
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
column_order: &mut Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> windmill_common::error::Result<Box<RawValue>> {
let bigquery_args = build_args_values(job, client, db).await?;
@@ -364,6 +366,7 @@ pub async fn do_bigquery(
result_f.map_err(to_anyhow),
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
)
.await?;
backend/windmill-worker/src/bun_executor.rs
+25 -4
View File
@@ -13,10 +13,11 @@ use crate::common::build_envs_map;
use crate::{
common::{
create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
parse_npm_config, read_file, read_file_content, read_result, start_child_process,
write_file_binary,
create_args_and_out_file, get_main_override, get_reserved_variables, parse_npm_config,
read_file, read_file_content, read_result, start_child_process, write_file_binary,
OccupancyMetrics,
},
handle_child::handle_child,
AuthedClientBackgroundTask, BUNFIG_INSTALL_SCOPES, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR,
BUN_DEPSTAR_CACHE_DIR, BUN_PATH, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NODE_BIN_PATH,
NODE_PATH, NPM_CONFIG_REGISTRY, NPM_PATH, NSJAIL_PATH, PATH_ENV, TZ_ENV,
@@ -69,6 +70,7 @@ pub async fn gen_bun_lockfile(
export_pkg: bool,
raw_deps: Option<String>,
npm_mode: bool,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> Result<Option<String>> {
let common_bun_proc_envs: HashMap<String, String> = get_common_bun_proc_envs(None).await;
@@ -125,6 +127,7 @@ pub async fn gen_bun_lockfile(
"bun build",
None,
false,
occupancy_metrics,
)
.await?;
} else {
@@ -149,6 +152,7 @@ pub async fn gen_bun_lockfile(
worker_name,
common_bun_proc_envs,
npm_mode,
occupancy_metrics,
)
.await?;
} else {
@@ -230,6 +234,7 @@ pub async fn install_bun_lockfile(
worker_name: &str,
common_bun_proc_envs: HashMap<String, String>,
npm_mode: bool,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> Result<()> {
let mut child_cmd = Command::new(if npm_mode { &*NPM_PATH } else { &*BUN_PATH });
child_cmd
@@ -296,6 +301,7 @@ pub async fn install_bun_lockfile(
"bun install",
None,
false,
occupancy_metrics,
)
.await?
} else {
@@ -436,6 +442,7 @@ pub async fn generate_wrapper_mjs(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
common_bun_proc_envs: &HashMap<String, String>,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> Result<()> {
let mut child = Command::new(&*BUN_PATH);
child
@@ -459,6 +466,7 @@ pub async fn generate_wrapper_mjs(
"bun build",
timeout,
false,
occupancy_metrics,
)
.await?;
fs::rename(
@@ -479,6 +487,7 @@ pub async fn generate_bun_bundle(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
common_bun_proc_envs: &HashMap<String, String>,
occupancy_metrics: &mut OccupancyMetrics,
) -> Result<()> {
let mut child = Command::new(&*BUN_PATH);
child
@@ -503,6 +512,7 @@ pub async fn generate_bun_bundle(
"bun build",
timeout,
false,
&mut Some(occupancy_metrics),
)
.await?;
} else {
@@ -609,6 +619,7 @@ pub async fn prebundle_bun_script(
base_internal_url: &str,
worker_name: &str,
token: &str,
occupancy_metrics: &mut OccupancyMetrics,
) -> Result<()> {
let (local_path, remote_path) = compute_bundle_local_and_remote_path(
inner_content,
@@ -656,6 +667,7 @@ pub async fn prebundle_bun_script(
&mut 0,
&mut None,
&common_bun_proc_envs,
occupancy_metrics,
)
.await?;
@@ -751,6 +763,7 @@ pub async fn handle_bun_job(
envs: HashMap<String, String>,
shared_mount: &str,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<Box<RawValue>> {
let mut annotation = windmill_common::worker::get_annotation(inner_content);
@@ -861,6 +874,7 @@ pub async fn handle_bun_job(
worker_name,
common_bun_proc_envs.clone(),
annotation.npm_mode,
&mut Some(occupancy_metrics),
)
.await?;
@@ -888,7 +902,6 @@ pub async fn handle_bun_job(
// if !*DISABLE_NSJAIL || !empty_trusted_deps || has_custom_config_registry {
let logs1 = "\n\n--- BUN INSTALL ---\n".to_string();
append_logs(&job.id, &job.workspace_id, logs1, db).await;
let _ = gen_bun_lockfile(
mem_peak,
canceled_by,
@@ -903,6 +916,7 @@ pub async fn handle_bun_job(
false,
None,
annotation.npm_mode,
&mut Some(occupancy_metrics),
)
.await?;
@@ -1128,6 +1142,7 @@ try {{
mem_peak,
canceled_by,
&common_bun_proc_envs,
occupancy_metrics,
)
.await?;
if !local_path.is_empty() {
@@ -1169,6 +1184,7 @@ try {{
mem_peak,
canceled_by,
&common_bun_proc_envs,
&mut Some(occupancy_metrics),
)
.await?;
}
@@ -1204,6 +1220,7 @@ try {{
worker_name,
&job.workspace_id,
false,
occupancy_metrics,
)
.await?;
tracing::info!(
@@ -1360,6 +1377,7 @@ try {{
"bun run",
job.timeout,
false,
&mut Some(occupancy_metrics),
)
.await?;
@@ -1501,6 +1519,7 @@ pub async fn start_worker(
worker_name,
common_bun_proc_envs.clone(),
annotation.npm_mode,
&mut None,
)
.await?;
tracing::info!("dedicated worker requirements installed: {reqs}");
@@ -1521,6 +1540,7 @@ pub async fn start_worker(
false,
None,
annotation.npm_mode,
&mut None,
)
.await?;
}
@@ -1617,6 +1637,7 @@ for await (const line of Readline.createInterface({{ input: process.stdin }})) {
&mut mem_peak,
&mut canceled_by,
&common_bun_proc_envs,
&mut None,
)
.await?;
}
backend/windmill-worker/src/common.rs
+65 -848
View File
@@ -1,15 +1,7 @@
use async_recursion::async_recursion;
use deno_ast::swc::parser::lexer::util::CharExt;
use futures::Future;
use itertools::Itertools;
#[cfg(any(target_os = "linux", target_os = "macos"))]
use nix::sys::signal::{self, Signal};
#[cfg(any(target_os = "linux", target_os = "macos"))]
use nix::unistd::Pid;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
use object_store::path::Path;
use regex::Regex;
use serde::{Deserialize, Serialize};
use serde_json::value::RawValue;
@@ -18,18 +10,15 @@ use sqlx::types::Json;
use sqlx::{Pool, Postgres};
use tokio::process::Command;
use tokio::{fs::File, io::AsyncReadExt};
use windmill_common::error::to_anyhow;
use windmill_common::jobs::ENTRYPOINT_OVERRIDE;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
#[cfg(feature = "parquet")]
use windmill_common::s3_helpers::{
get_etag_or_empty, LargeFileStorage, ObjectStoreResource, S3Object,
};
use windmill_common::variables::{build_crypt_with_key_suffix, decrypt_value_with_mc};
use windmill_common::worker::{
get_windmill_memory_usage, get_worker_memory_usage, to_raw_value, write_file, CLOUD_HOSTED,
ROOT_CACHE_DIR, TMP_DIR, WORKER_CONFIG,
to_raw_value, write_file, CLOUD_HOSTED, ROOT_CACHE_DIR, WORKER_CONFIG,
};
use windmill_common::{
error::{self, Error},
@@ -38,43 +27,21 @@ use windmill_common::{
};
use anyhow::{anyhow, Result};
use windmill_queue::{append_logs, CanceledBy};
#[cfg(any(target_os = "linux", target_os = "macos"))]
use std::os::unix::process::ExitStatusExt;
use std::process::ExitStatus;
use std::sync::atomic::AtomicU32;
use std::sync::Arc;
use std::{
collections::{hash_map::DefaultHasher, HashMap},
hash::{Hash, Hasher},
io, panic,
time::Duration,
};
use tracing::{trace_span, Instrument};
use uuid::Uuid;
use windmill_common::{variables, DB};
#[cfg(feature = "enterprise")]
use windmill_common::job_metrics;
use tokio::{
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
process::Child,
sync::{broadcast, watch},
time::{interval, sleep, Instant, MissedTickBehavior},
};
use futures::{
future::{self, ready, FutureExt},
stream, StreamExt,
};
use tokio::{io::AsyncWriteExt, process::Child, time::Instant};
use crate::{
AuthedClient, AuthedClientBackgroundTask, JOB_DEFAULT_TIMEOUT, MAX_RESULT_SIZE,
MAX_TIMEOUT_DURATION, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM,
MAX_TIMEOUT_DURATION,
};
pub async fn build_args_map<'a>(
@@ -466,36 +433,6 @@ pub fn get_main_override(args: Option<&Json<HashMap<String, Box<RawValue>>>>) ->
.flatten();
}
async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
if pid.is_none() {
return -1;
}
let pid = if nsjail {
// This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1.
// Ideally, we would get the number from fork() itself. This works in MOST cases.
pid.unwrap() + 1
} else {
pid.unwrap()
};
if let Ok(file) = File::open(format!("/proc/{}/status", pid)).await {
let mut lines = BufReader::new(file).lines();
while let Some(line) = lines.next_line().await.unwrap_or(None) {
if line.starts_with("VmHWM:") {
return line
.split_whitespace()
.nth(1)
.and_then(|s| s.parse::<i32>().ok())
.unwrap_or(-1);
};
}
-2
} else {
// rand::random::<i32>() % 100 // to remove - used to fake memory data on MacOS
-3
}
}
pub fn sizeof_val(v: &serde_json::Value) -> usize {
std::mem::size_of::<serde_json::Value>()
+ match v {
@@ -516,63 +453,6 @@ pub fn sizeof_val(v: &serde_json::Value) -> usize {
}
}
pub async fn run_future_with_polling_update_job_poller<Fut, T>(
job_id: Uuid,
timeout: Option<i32>,
db: &DB,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
result_f: Fut,
worker_name: &str,
w_id: &str,
) -> error::Result<T>
where
Fut: Future<Output = anyhow::Result<T>>,
{
let (tx, rx) = broadcast::channel::<()>(3);
let update_job = update_job_poller(
job_id,
db,
mem_peak,
canceled_by_ref,
|| async { 0 },
worker_name,
w_id,
rx,
);
let timeout_ms = u64::try_from(
resolve_job_timeout(&db, &w_id, job_id, timeout)
.await
.0
.as_millis(),
)
.unwrap_or(200000);
let rows = tokio::select! {
biased;
result = tokio::time::timeout(std::time::Duration::from_millis(timeout_ms), result_f) => result
.map_err(|e| {
tracing::error!("Query timeout: {}", e);
Error::ExecutionErr(format!("Query timeout after (>{}s)", timeout_ms/1000))
})?,
ex = update_job, if job_id != Uuid::nil() => {
match ex {
UpdateJobPollingExit::Done => Err(Error::ExecutionErr("Job cancelled".to_string())).map_err(to_anyhow)?,
UpdateJobPollingExit::AlreadyCompleted => Err(Error::AlreadyCompleted("Job already completed".to_string())).map_err(to_anyhow)?,
}
}
}?;
drop(tx);
Ok(rows)
}
pub enum UpdateJobPollingExit {
Done,
AlreadyCompleted,
}
pub async fn update_worker_ping_for_failed_init_script(
db: &DB,
worker_name: &str,
@@ -594,672 +474,83 @@ pub async fn update_worker_ping_for_failed_init_script(
tracing::error!("Error updating worker ping for failed init script: {e:?}");
}
}
pub struct OccupancyMetrics {
pub running_job_started_at: Option<Instant>,
pub total_duration_of_running_jobs: f32,
pub worker_occupancy_rate_history: Vec<(f32, f32)>,
pub start_time: Instant,
}
pub async fn update_job_poller<F, Fut>(
job_id: Uuid,
db: &DB,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
get_mem: F,
worker_name: &str,
w_id: &str,
mut rx: broadcast::Receiver<()>,
) -> UpdateJobPollingExit
where
F: Fn() -> Fut,
Fut: Future<Output = i32>,
{
let update_job_interval = Duration::from_millis(500);
impl OccupancyMetrics {
pub fn new(start_time: Instant) -> Self {
OccupancyMetrics {
running_job_started_at: None,
total_duration_of_running_jobs: 0.0,
worker_occupancy_rate_history: Vec::new(),
start_time,
}
}
let db = db.clone();
pub fn update_occupancy_metrics(&mut self) -> (f32, Option<f32>, Option<f32>, Option<f32>) {
let metrics = self;
let current_occupied_duration = metrics
.running_job_started_at
.map(|started_at| started_at.elapsed().as_secs_f32())
.unwrap_or(0.0);
let total_occupation = metrics.total_duration_of_running_jobs + current_occupied_duration;
let mut interval = interval(update_job_interval);
interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
let elapsed = metrics.start_time.elapsed().as_secs_f32();
let mut i = 0;
#[cfg(feature = "enterprise")]
let mut memory_metric_id: Result<String, Error> =
Err(Error::NotFound("not yet initialized".to_string()));
loop {
tokio::select!(
_ = rx.recv() => break,
_ = interval.tick() => {
// update the last_ping column every 5 seconds
i+=1;
if i == 1 || i % 10 == 0 {
let memory_usage = get_worker_memory_usage();
let wm_memory_usage = get_windmill_memory_usage();
tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024);
if job_id != Uuid::nil() {
sqlx::query!(
"UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4 WHERE worker = $5",
&job_id,
&w_id,
memory_usage,
wm_memory_usage,
&worker_name
)
.execute(&db)
.await
.expect("update worker ping");
}
}
let current_mem = get_mem().await;
if current_mem > *mem_peak {
*mem_peak = current_mem
}
tracing::info!("job {job_id} on {worker_name} in {w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB");
let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0;
if update_job_row {
#[cfg(feature = "enterprise")]
let (occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m) =
if !metrics.worker_occupancy_rate_history.is_empty() {
let mut total_occupation_15s = 0.0;
let mut total_occupation_5m = 0.0;
let mut total_occupation_30m = 0.0;
let mut index30m = 0;
for (i, (past_total_occupation, time)) in
metrics.worker_occupancy_rate_history.iter().enumerate()
{
if job_id != Uuid::nil() {
// tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
if i == 2 {
memory_metric_id = job_metrics::register_metric_for_job(
&db,
w_id.to_string(),
job_id,
"memory_kb".to_string(),
job_metrics::MetricKind::TimeseriesInt,
Some("Job Memory Footprint (kB)".to_string()),
)
.await;
}
if let Ok(ref metric_id) = memory_metric_id {
if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
}
}
let diff = elapsed - time;
if diff < 1800.0 && total_occupation_30m == 0.0 {
total_occupation_30m = (total_occupation - past_total_occupation) / diff;
index30m = i;
}
if diff < 300.0 && total_occupation_5m == 0.0 {
total_occupation_5m = (total_occupation - past_total_occupation) / diff;
}
if diff < 15.0 {
total_occupation_15s = (total_occupation - past_total_occupation) / diff;
break;
}
}
if job_id != Uuid::nil() {
let (canceled, canceled_by, canceled_reason, already_completed) = sqlx::query_as::<_, (bool, Option<String>, Option<String>, bool)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason, false")
.bind(*mem_peak)
.bind(job_id)
.fetch_optional(&db)
.await
.unwrap_or_else(|e| {
tracing::error!(%e, "error updating job {job_id}: {e:#}");
Some((false, None, None, false))
})
.unwrap_or_else(|| {
// if the job is not in queue, it can only be in the completed_job so it is already complete
(false, None, None, true)
});
if already_completed {
return UpdateJobPollingExit::AlreadyCompleted
}
if canceled {
canceled_by_ref.replace(CanceledBy {
username: canceled_by.clone(),
reason: canceled_reason.clone(),
});
break
}
}
}
},
);
}
tracing::info!("job {job_id} finished");
UpdateJobPollingExit::Done
}
//drop all elements before the oldest one in 30m windows
metrics.worker_occupancy_rate_history.drain(..index30m);
pub enum CompactLogs {
NotEE,
NoS3,
S3,
}
async fn compact_logs(
job_id: Uuid,
w_id: &str,
db: &DB,
nlogs: String,
total_size: Arc<AtomicU32>,
compact_kind: CompactLogs,
_worker_name: &str,
) -> error::Result<(String, String)> {
let mut prev_logs = sqlx::query_scalar!(
"SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2",
job_id,
w_id
)
.fetch_optional(db)
.await?
.flatten()
.unwrap_or_default();
let size = prev_logs.char_indices().count() as i32;
let nlogs_len = nlogs.char_indices().count();
let to_keep_in_db = usize::max(
usize::min(nlogs_len, 3000),
nlogs_len % LARGE_LOG_THRESHOLD_SIZE,
);
let extra_split = to_keep_in_db < nlogs_len;
let stored_in_storage_len = if extra_split {
nlogs_len - to_keep_in_db
} else {
0
};
let extra_to_newline = nlogs
.chars()
.skip(stored_in_storage_len)
.find_position(|x| x.is_line_break())
.map(|(i, _)| i)
.unwrap_or(to_keep_in_db);
let stored_in_storage_to_newline = stored_in_storage_len + extra_to_newline;
let (append_to_storage, stored_in_db) = if extra_split {
if stored_in_storage_to_newline == nlogs.len() {
(nlogs.as_ref(), "".to_string())
} else {
let split_idx = nlogs
.char_indices()
.nth(stored_in_storage_to_newline)
.map(|(i, _)| i)
.unwrap_or(0);
let (append_to_storage, stored_in_db) = nlogs.split_at(split_idx);
// tracing::error!("{append_to_storage} ||||| {stored_in_db}");
// tracing::error!(
// "{:?} {:?} {} {}",
// excess_prev_logs.lines().last(),
// current_logs.lines().next(),
// split_idx,
// excess_size_modulo
// );
(append_to_storage, stored_in_db.to_string())
}
} else {
// tracing::error!("{:?}", nlogs.lines().last());
("", nlogs.to_string())
};
let new_size_with_excess = size + stored_in_storage_to_newline as i32;
let new_size = total_size.fetch_add(
new_size_with_excess as u32,
std::sync::atomic::Ordering::SeqCst,
) + new_size_with_excess as u32;
let path = format!(
"logs/{job_id}/{}_{new_size}.txt",
chrono::Utc::now().timestamp_millis()
);
let mut new_current_logs = match compact_kind {
CompactLogs::NoS3 => format!("\n[windmill] No object storage set in instance settings. Previous logs have been saved to disk at {path}"),
CompactLogs::S3 => format!("\n[windmill] Previous logs have been saved to object storage at {path}"),
CompactLogs::NotEE => format!("\n[windmill] Previous logs have been saved to disk at {path}"),
};
new_current_logs.push_str(&stored_in_db);
sqlx::query!(
"UPDATE job_logs SET logs = $1, log_offset = $2,
log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3)
WHERE workspace_id = $4 AND job_id = $5",
new_current_logs,
new_size as i32,
path,
w_id,
job_id
)
.execute(db)
.await?;
prev_logs.push_str(&append_to_storage);
return Ok((prev_logs, path));
}
async fn default_disk_log_storage(
job_id: Uuid,
w_id: &str,
db: &DB,
nlogs: String,
total_size: Arc<AtomicU32>,
compact_kind: CompactLogs,
worker_name: &str,
) {
match compact_logs(
job_id,
&w_id,
&db,
nlogs,
total_size,
compact_kind,
worker_name,
)
.await
{
Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
Ok((prev_logs, path)) => {
let path = format!("{}/{}", TMP_DIR, path);
let splitted = &path.split("/").collect_vec();
tokio::fs::create_dir_all(splitted.into_iter().take(splitted.len() - 1).join("/"))
.await
.map_err(|e| {
tracing::error!("Could not create logs directory: {e:?}",);
e
})
.ok();
let created = tokio::fs::File::create(&path).await;
if let Err(e) = created {
tracing::error!("Could not create logs file {path}: {e:?}",);
return;
}
if let Err(e) = tokio::fs::write(&path, prev_logs).await {
tracing::error!("Could not write to logs file {path}: {e:?}");
} else {
tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to disk at {path}");
}
}
}
}
async fn append_job_logs(
job_id: Uuid,
w_id: String,
logs: String,
db: DB,
must_compact_logs: bool,
total_size: Arc<AtomicU32>,
worker_name: String,
) -> () {
if must_compact_logs {
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
match compact_logs(
job_id,
&w_id,
&db,
logs,
total_size,
CompactLogs::S3,
&worker_name,
)
.await
{
Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
Ok((prev_logs, path)) => {
tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to object storage at {path}");
let path2 = path.clone();
if let Err(e) = os
.put(&Path::from(path), prev_logs.to_string().into_bytes().into())
.await
{
tracing::error!("Could not save logs to s3: {e:?}");
}
tracing::info!("Logs of {job_id} saved to object storage at {path2}");
}
}
} else {
default_disk_log_storage(
job_id,
&w_id,
&db,
logs,
total_size,
CompactLogs::NoS3,
&worker_name,
)
.await;
}
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
{
default_disk_log_storage(
job_id,
&w_id,
&db,
logs,
total_size,
CompactLogs::NotEE,
&worker_name,
)
.await;
}
} else {
append_logs(&job_id, w_id, logs, db).await;
}
}
pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000;
/// - wait until child exits and return with exit status
/// - read lines from stdout and stderr and append them to the "queue"."logs"
/// quitting early if output exceedes MAX_LOG_SIZE characters (not bytes)
/// - update the `last_line` and `logs` strings with the program output
/// - update "queue"."last_ping" every five seconds
/// - kill process if we exceed timeout or "queue"."canceled" is set
#[tracing::instrument(level = "trace", skip_all)]
pub async fn handle_child(
job_id: &Uuid,
db: &Pool<Postgres>,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
mut child: Child,
nsjail: bool,
worker: &str,
w_id: &str,
child_name: &str,
custom_timeout: Option<i32>,
sigterm: bool,
) -> error::Result<()> {
let start = Instant::now();
let pid = child.id();
#[cfg(target_os = "linux")]
if let Some(pid) = pid {
//set the highest oom priority
if let Some(mut file) = File::create(format!("/proc/{pid}/oom_score_adj"))
.await
.map_err(|e| {
tracing::error!("Could not create oom_score_file to pid {pid}: {e:#}");
e
})
.ok()
{
let _ = file.write_all(b"1000").await;
let _ = file.sync_all().await;
}
} else {
tracing::info!("could not get child pid");
}
let (set_too_many_logs, mut too_many_logs) = watch::channel::<bool>(false);
let (tx, rx) = broadcast::channel::<()>(3);
let mut rx2 = tx.subscribe();
let output = child_joined_output_stream(&mut child);
let job_id = job_id.clone();
/* the cancellation future is polled on by `wait_on_child` while
* waiting for the child to exit normally */
let update_job = update_job_poller(
job_id,
db,
mem_peak,
canceled_by_ref,
|| get_mem_peak(pid, nsjail),
worker,
w_id,
rx,
);
#[derive(PartialEq, Debug)]
enum KillReason {
TooManyLogs,
Timeout,
Cancelled,
AlreadyCompleted,
}
let (timeout_duration, timeout_warn_msg) =
resolve_job_timeout(&db, w_id, job_id, custom_timeout).await;
if let Some(msg) = timeout_warn_msg {
append_logs(&job_id, w_id, msg.as_str(), db).await;
}
/* a future that completes when the child process exits */
let wait_on_child = async {
let db = db.clone();
let kill_reason = tokio::select! {
biased;
result = child.wait() => return result.map(Ok),
Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
_ = sleep(timeout_duration) => KillReason::Timeout,
ex = update_job, if job_id != Uuid::nil() => match ex {
UpdateJobPollingExit::Done => KillReason::Cancelled,
UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted,
},
};
tx.send(()).expect("rx should never be dropped");
drop(tx);
let set_reason = async {
if kill_reason == KillReason::Timeout {
if let Err(err) = sqlx::query(
r#"
UPDATE queue
SET canceled = true
, canceled_by = 'timeout'
, canceled_reason = $1
WHERE id = $2
"#,
(
Some(total_occupation_15s),
Some(total_occupation_5m),
Some(total_occupation_30m),
)
.bind(format!("duration > {}", timeout_duration.as_secs()))
.bind(job_id)
.execute(&db)
.await
{
tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}");
}
}
};
if let Some(id) = child.id() {
if *MAX_WAIT_FOR_SIGINT > 0 {
#[cfg(any(target_os = "linux", target_os = "macos"))]
signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap();
for _ in 0..*MAX_WAIT_FOR_SIGINT {
if child.try_wait().is_ok_and(|x| x.is_some()) {
break;
}
sleep(Duration::from_secs(1)).await;
}
if child.try_wait().is_ok_and(|x| x.is_some()) {
set_reason.await;
return Ok(Err(kill_reason));
}
}
if sigterm {
#[cfg(any(target_os = "linux", target_os = "macos"))]
signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap();
for _ in 0..*MAX_WAIT_FOR_SIGTERM {
if child.try_wait().is_ok_and(|x| x.is_some()) {
break;
}
sleep(Duration::from_secs(1)).await;
}
if child.try_wait().is_ok_and(|x| x.is_some()) {
set_reason.await;
return Ok(Err(kill_reason));
}
}
}
/* send SIGKILL and reap child process */
let (_, kill) = future::join(set_reason, child.kill()).await;
kill.map(|()| Err(kill_reason))
};
/* a future that reads output from the child and appends to the database */
let lines = async move {
let max_log_size = if *CLOUD_HOSTED {
MAX_RESULT_SIZE
} else {
usize::MAX
};
/* log_remaining is zero when output limit was reached */
let mut log_remaining = if *CLOUD_HOSTED {
max_log_size
} else {
usize::MAX
};
let mut result = io::Result::Ok(());
let mut output = output.take_until(async {
let _ = rx2.recv().await;
//wait at most 50ms after end of a script for output stream to end
tokio::time::sleep(Duration::from_millis(50)).await;
}).boxed();
/* `do_write` resolves the task, but does not contain the Result.
* It's useful to know if the task completed. */
let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle();
let mut log_total_size: u64 = 0;
let pg_log_total_size = Arc::new(AtomicU32::new(0));
while let Some(line) = output.by_ref().next().await {
let do_write_ = do_write.shared();
let delay = if start.elapsed() < Duration::from_secs(10) {
Duration::from_millis(500)
} else if start.elapsed() < Duration::from_secs(60){
Duration::from_millis(2500)
} else {
Duration::from_millis(5000)
(None, None, None)
};
let occupancy_rate = total_occupation / elapsed;
let delay = if *SLOW_LOGS {
delay * 10
} else {
delay
};
//push the current occupancy rate and the timestamp
metrics
.worker_occupancy_rate_history
.push((total_occupation, elapsed));
let mut read_lines = stream::once(async { line })
.chain(output.by_ref())
/* after receiving a line, continue until some delay has passed
* _and_ the previous database write is complete */
.take_until(future::join(sleep(delay), do_write_.clone()))
.boxed();
/* Read up until an error is encountered,
* handle log lines first and then the error... */
let mut joined = String::new();
while let Some(line) = read_lines.next().await {
match line {
Ok(line) => {
if line.is_empty() {
continue;
}
append_with_limit(&mut joined, &line, &mut log_remaining);
if log_remaining == 0 {
tracing::info!(%job_id, "Too many logs lines for job {job_id}");
let _ = set_too_many_logs.send(true);
joined.push_str(&format!(
"Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job."
));
/* stop reading and drop our streams fairly quickly */
break;
}
}
Err(err) => {
result = Err(err);
break;
}
}
}
/* Ensure the last flush completed before starting a new one.
*
* This shouldn't pause since `take_until()` reads lines until `do_write`
* resolves. We only stop reading lines before `take_until()` resolves if we reach
* EOF or a read error. In those cases, waiting on a database query to complete is
* fine because we're done. */
if let Some(Ok(p)) = do_write_
.then(|()| write_result)
.await
.err()
.map(|err| err.try_into_panic())
{
panic::resume_unwind(p);
}
let joined_len = joined.len() as u64;
log_total_size += joined_len;
let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64;
if compact_logs {
log_total_size = 0;
}
let worker_name = worker.to_string();
let w_id2 = w_id.to_string();
(do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
if let Err(err) = result {
tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}");
break;
}
if *set_too_many_logs.borrow() {
break;
}
}
/* drop our end of the pipe */
drop(output);
if let Some(Ok(p)) = do_write
.then(|()| write_result)
.await
.err()
.map(|err| err.try_into_panic())
{
panic::resume_unwind(p);
}
}.instrument(trace_span!("child_lines"));
let (wait_result, _) = tokio::join!(wait_on_child, lines);
let success = wait_result.is_ok()
&& wait_result.as_ref().unwrap().is_ok()
&& wait_result.as_ref().unwrap().as_ref().unwrap().success();
tracing::info!(%job_id, %success, %mem_peak, %worker, "child process '{child_name}' took {}ms", start.elapsed().as_millis());
match wait_result {
_ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
"logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)"
))),
Ok(Ok(status)) => process_status(status),
Ok(Err(kill_reason)) => match kill_reason {
KillReason::AlreadyCompleted => {
Err(Error::AlreadyCompleted("Job already completed".to_string()))
}
_ => Err(Error::ExecutionErr(format!(
"job process killed because {kill_reason:#?}"
))),
},
Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))),
(
occupancy_rate,
occupancy_rate_15s,
occupancy_rate_5m,
occupancy_rate_30m,
)
}
}
pub fn process_status(status: ExitStatus) -> error::Result<()> {
if status.success() {
Ok(())
} else if let Some(code) = status.code() {
Err(error::Error::ExitStatus(code))
} else {
#[cfg(any(target_os = "linux", target_os = "macos"))]
return Err(error::Error::ExecutionErr(format!(
"process terminated by signal: {:#?}, stopped_signal: {:#?}, core_dumped: {}",
status.signal(),
status.stopped_signal(),
status.core_dumped()
)));
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
return Err(error::Error::ExecutionErr(String::from(
"process terminated by signal",
)));
}
}
pub async fn start_child_process(mut cmd: Command, executable: &str) -> Result<Child, Error> {
return cmd
.spawn()
@@ -1324,80 +615,6 @@ pub async fn resolve_job_timeout(
}
}
/// takes stdout and stderr from Child, panics if either are not present
///
/// builds a stream joining both stdout and stderr each read line by line
fn child_joined_output_stream(
child: &mut Child,
) -> impl stream::FusedStream<Item = io::Result<String>> {
let stderr = child
.stderr
.take()
.expect("child did not have a handle to stdout");
let stdout = child
.stdout
.take()
.expect("child did not have a handle to stdout");
let stdout = BufReader::new(stdout).lines();
let stderr = BufReader::new(stderr).lines();
stream::select(lines_to_stream(stderr), lines_to_stream(stdout))
}
pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
mut lines: tokio::io::Lines<R>,
) -> impl futures::Stream<Item = io::Result<String>> {
stream::poll_fn(move |cx| {
std::pin::Pin::new(&mut lines)
.poll_next_line(cx)
.map(|result| result.transpose())
})
}
lazy_static::lazy_static! {
static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap();
pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true");
pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true");
}
// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields,
// so this pushes \n to thd destination string in each call
fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
if *NO_LOGS_AT_ALL {
return;
}
let src_str;
let src = {
src_str = RE_00.replace_all(src, "");
src_str.as_ref()
};
if !*CLOUD_HOSTED {
dst.push('\n');
dst.push_str(&src);
return;
} else {
if *limit > 0 {
dst.push('\n');
}
*limit -= 1;
}
let src_len = src.chars().count();
if src_len <= *limit {
dst.push_str(&src);
*limit -= src_len;
} else {
let byte_pos = src
.char_indices()
.skip(*limit)
.next()
.map(|(byte_pos, _)| byte_pos)
.unwrap_or(0);
dst.push_str(&src[0..byte_pos]);
*limit = 0;
}
}
pub async fn hash_args(
_db: &DB,
_client: &AuthedClient,
backend/windmill-worker/src/dedicated_worker.rs
+4 -3
View File
@@ -29,8 +29,7 @@ use windmill_queue::append_logs;
use anyhow::Context;
use crate::{
common::{process_status, start_child_process},
JobCompleted, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS,
common::start_child_process, JobCompleted, JobCompletedSender, MAX_BUFFERED_DEDICATED_JOBS,
};
use futures::{future, Future};
@@ -78,6 +77,8 @@ pub async fn handle_dedicated_process(
) -> std::result::Result<(), error::Error> {
//do not cache local dependencies
use crate::handle_child::process_status;
let mut child = {
let mut cmd = Command::new(command_path);
cmd.current_dir(job_dir)
@@ -127,7 +128,7 @@ pub async fn handle_dedicated_process(
if let Err(e) = process_status(status) {
tracing::error!("child exit status was not success: {e:#}");
} else {
tracing::info!("child exist status was success");
tracing::info!("child exit status was success");
}
});
backend/windmill-worker/src/deno_executor.rs
+7 -2
View File
@@ -7,9 +7,10 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
parse_npm_config, read_file, read_result, start_child_process,
create_args_and_out_file, get_main_override, get_reserved_variables, parse_npm_config,
read_file, read_result, start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
AuthedClientBackgroundTask, DENO_CACHE_DIR, DENO_PATH, DISABLE_NSJAIL, HOME_ENV,
NPM_CONFIG_REGISTRY, PATH_ENV, TZ_ENV,
};
@@ -94,6 +95,7 @@ pub async fn generate_deno_lock(
w_id: &str,
worker_name: &str,
base_internal_url: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<String> {
let _ = write_file(job_dir, "main.ts", code)?;
@@ -146,6 +148,7 @@ pub async fn generate_deno_lock(
"deno cache",
None,
false,
occupancy_metrics,
)
.await?;
} else {
@@ -173,6 +176,7 @@ pub async fn handle_deno_job(
worker_name: &str,
envs: HashMap<String, String>,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<Box<RawValue>> {
// let mut start = Instant::now();
let logs1 = "\n\n--- DENO CODE EXECUTION ---\n".to_string();
@@ -397,6 +401,7 @@ try {{
"deno run",
job.timeout,
false,
&mut Some(occupancy_metrics),
)
.await?;
// logs.push_str(format!("execute: {:?}\n", start.elapsed().as_millis()).as_str());
backend/windmill-worker/src/go_executor.rs
+11 -4
View File
@@ -15,9 +15,10 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
capitalize, create_args_and_out_file, get_reserved_variables, handle_child, read_result,
start_child_process,
capitalize, create_args_and_out_file, get_reserved_variables, read_result,
start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, GOPRIVATE, GOPROXY,
GO_BIN_CACHE_DIR, GO_CACHE_DIR, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV,
};
@@ -44,6 +45,7 @@ pub async fn handle_go_job(
base_internal_url: &str,
worker_name: &str,
envs: HashMap<String, String>,
occupation_metrics: &mut OccupancyMetrics,
) -> Result<Box<RawValue>, Error> {
//go does not like executing modules at temp root
let job_dir = &format!("{job_dir}/go");
@@ -88,6 +90,7 @@ pub async fn handle_go_job(
skip_tidy,
worker_name,
&job.workspace_id,
occupation_metrics,
)
.await?;
@@ -202,6 +205,7 @@ func Run(req Req) (interface{{}}, error){{
"go build",
None,
false,
&mut Some(occupation_metrics),
)
.await?;
@@ -297,6 +301,7 @@ func Run(req Req) (interface{{}}, error){{
"go run",
job.timeout,
false,
&mut Some(occupation_metrics),
)
.await?;
@@ -336,6 +341,7 @@ pub async fn install_go_dependencies(
has_sum: bool,
worker_name: &str,
w_id: &str,
occupation_metrics: &mut OccupancyMetrics,
) -> error::Result<String> {
if !skip_go_mod {
gen_go_mymod(code, job_dir).await?;
@@ -359,6 +365,7 @@ pub async fn install_go_dependencies(
"go init",
None,
false,
&mut Some(occupation_metrics),
)
.await?;
@@ -424,9 +431,9 @@ pub async fn install_go_dependencies(
&format!("go {mod_command}"),
None,
false,
&mut Some(occupation_metrics),
)
.await
.map_err(|e| Error::ExecutionErr(format!("Lockfile generation failed: {e:?}")))?;
.await?;
if (!new_lockfile || has_sum) && non_dep_job {
return Ok("".to_string());
backend/windmill-worker/src/graphql_executor.rs
+4 -1
View File
@@ -12,7 +12,8 @@ use windmill_queue::{CanceledBy, HTTP_CLIENT};
use serde::Deserialize;
use crate::common::run_future_with_polling_update_job_poller;
use crate::common::OccupancyMetrics;
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::{common::build_args_map, AuthedClientBackgroundTask};
#[derive(Deserialize)]
@@ -41,6 +42,7 @@ pub async fn do_graphql(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
occupation_metrics: &mut OccupancyMetrics,
) -> windmill_common::error::Result<Box<RawValue>> {
let args = build_args_map(job, client, db).await?.map(Json);
let job_args = if args.is_some() {
@@ -151,6 +153,7 @@ pub async fn do_graphql(
result_f,
worker_name,
&job.workspace_id,
&mut Some(occupation_metrics),
)
.await?;
backend/windmill-worker/src/handle_child.rs
+627
View File
@@ -0,0 +1,627 @@
use futures::Future;
#[cfg(any(target_os = "linux", target_os = "macos"))]
use nix::sys::signal::{self, Signal};
#[cfg(any(target_os = "linux", target_os = "macos"))]
use nix::unistd::Pid;
use sqlx::{Pool, Postgres};
use tokio::fs::File;
use windmill_common::error::to_anyhow;
use windmill_common::error::{self, Error};
use windmill_common::worker::{get_windmill_memory_usage, get_worker_memory_usage, CLOUD_HOSTED};
use windmill_queue::{append_logs, CanceledBy};
#[cfg(any(target_os = "linux", target_os = "macos"))]
use std::os::unix::process::ExitStatusExt;
use std::process::ExitStatus;
use std::sync::atomic::AtomicU32;
use std::sync::Arc;
use std::{io, panic, time::Duration};
use tracing::{trace_span, Instrument};
use uuid::Uuid;
use windmill_common::DB;
#[cfg(feature = "enterprise")]
use windmill_common::job_metrics;
use tokio::{
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
process::Child,
sync::{broadcast, watch},
time::{interval, sleep, Instant, MissedTickBehavior},
};
use futures::{
future::{self, ready, FutureExt},
stream, StreamExt,
};
use crate::common::{resolve_job_timeout, OccupancyMetrics};
use crate::job_logger::{append_job_logs, append_with_limit, LARGE_LOG_THRESHOLD_SIZE};
use crate::{MAX_RESULT_SIZE, MAX_WAIT_FOR_SIGINT, MAX_WAIT_FOR_SIGTERM};
lazy_static::lazy_static! {
pub static ref SLOW_LOGS: bool = std::env::var("SLOW_LOGS").ok().is_some_and(|x| x == "1" || x == "true");
}
/// - wait until child exits and return with exit status
/// - read lines from stdout and stderr and append them to the "queue"."logs"
/// quitting early if output exceedes MAX_LOG_SIZE characters (not bytes)
/// - update the `last_line` and `logs` strings with the program output
/// - update "queue"."last_ping" every five seconds
/// - kill process if we exceed timeout or "queue"."canceled" is set
#[tracing::instrument(level = "trace", skip_all)]
pub async fn handle_child(
job_id: &Uuid,
db: &Pool<Postgres>,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
mut child: Child,
nsjail: bool,
worker: &str,
w_id: &str,
child_name: &str,
custom_timeout: Option<i32>,
sigterm: bool,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<()> {
let start = Instant::now();
let pid = child.id();
#[cfg(target_os = "linux")]
if let Some(pid) = pid {
//set the highest oom priority
if let Some(mut file) = File::create(format!("/proc/{pid}/oom_score_adj"))
.await
.map_err(|e| {
tracing::error!("Could not create oom_score_file to pid {pid}: {e:#}");
e
})
.ok()
{
let _ = file.write_all(b"1000").await;
let _ = file.sync_all().await;
}
} else {
tracing::info!("could not get child pid");
}
let (set_too_many_logs, mut too_many_logs) = watch::channel::<bool>(false);
let (tx, rx) = broadcast::channel::<()>(3);
let mut rx2 = tx.subscribe();
let output = child_joined_output_stream(&mut child);
let job_id = job_id.clone();
/* the cancellation future is polled on by `wait_on_child` while
* waiting for the child to exit normally */
let update_job = update_job_poller(
job_id,
db,
mem_peak,
canceled_by_ref,
|| get_mem_peak(pid, nsjail),
worker,
w_id,
rx,
occupancy_metrics,
);
#[derive(PartialEq, Debug)]
enum KillReason {
TooManyLogs,
Timeout,
Cancelled,
AlreadyCompleted,
}
let (timeout_duration, timeout_warn_msg) =
resolve_job_timeout(&db, w_id, job_id, custom_timeout).await;
if let Some(msg) = timeout_warn_msg {
append_logs(&job_id, w_id, msg.as_str(), db).await;
}
/* a future that completes when the child process exits */
let wait_on_child = async {
let db = db.clone();
let kill_reason = tokio::select! {
biased;
result = child.wait() => return result.map(Ok),
Ok(()) = too_many_logs.changed() => KillReason::TooManyLogs,
_ = sleep(timeout_duration) => KillReason::Timeout,
ex = update_job, if job_id != Uuid::nil() => match ex {
UpdateJobPollingExit::Done => KillReason::Cancelled,
UpdateJobPollingExit::AlreadyCompleted => KillReason::AlreadyCompleted,
},
};
tx.send(()).expect("rx should never be dropped");
drop(tx);
let set_reason = async {
if kill_reason == KillReason::Timeout {
if let Err(err) = sqlx::query(
r#"
UPDATE queue
SET canceled = true
, canceled_by = 'timeout'
, canceled_reason = $1
WHERE id = $2
"#,
)
.bind(format!("duration > {}", timeout_duration.as_secs()))
.bind(job_id)
.execute(&db)
.await
{
tracing::error!(%job_id, %err, "error setting cancelation reason for job {job_id}: {err}");
}
}
};
if let Some(id) = child.id() {
if *MAX_WAIT_FOR_SIGINT > 0 {
#[cfg(any(target_os = "linux", target_os = "macos"))]
signal::kill(Pid::from_raw(id as i32), Signal::SIGINT).unwrap();
for _ in 0..*MAX_WAIT_FOR_SIGINT {
if child.try_wait().is_ok_and(|x| x.is_some()) {
break;
}
sleep(Duration::from_secs(1)).await;
}
if child.try_wait().is_ok_and(|x| x.is_some()) {
set_reason.await;
return Ok(Err(kill_reason));
}
}
if sigterm {
#[cfg(any(target_os = "linux", target_os = "macos"))]
signal::kill(Pid::from_raw(id as i32), Signal::SIGTERM).unwrap();
for _ in 0..*MAX_WAIT_FOR_SIGTERM {
if child.try_wait().is_ok_and(|x| x.is_some()) {
break;
}
sleep(Duration::from_secs(1)).await;
}
if child.try_wait().is_ok_and(|x| x.is_some()) {
set_reason.await;
return Ok(Err(kill_reason));
}
}
}
/* send SIGKILL and reap child process */
let (_, kill) = future::join(set_reason, child.kill()).await;
kill.map(|()| Err(kill_reason))
};
/* a future that reads output from the child and appends to the database */
let lines = async move {
let max_log_size = if *CLOUD_HOSTED {
MAX_RESULT_SIZE
} else {
usize::MAX
};
/* log_remaining is zero when output limit was reached */
let mut log_remaining = if *CLOUD_HOSTED {
max_log_size
} else {
usize::MAX
};
let mut result = io::Result::Ok(());
let mut output = output.take_until(async {
let _ = rx2.recv().await;
//wait at most 50ms after end of a script for output stream to end
tokio::time::sleep(Duration::from_millis(50)).await;
}).boxed();
/* `do_write` resolves the task, but does not contain the Result.
* It's useful to know if the task completed. */
let (mut do_write, mut write_result) = tokio::spawn(ready(())).remote_handle();
let mut log_total_size: u64 = 0;
let pg_log_total_size = Arc::new(AtomicU32::new(0));
while let Some(line) = output.by_ref().next().await {
let do_write_ = do_write.shared();
let delay = if start.elapsed() < Duration::from_secs(10) {
Duration::from_millis(500)
} else if start.elapsed() < Duration::from_secs(60){
Duration::from_millis(2500)
} else {
Duration::from_millis(5000)
};
let delay = if *SLOW_LOGS {
delay * 10
} else {
delay
};
let mut read_lines = stream::once(async { line })
.chain(output.by_ref())
/* after receiving a line, continue until some delay has passed
* _and_ the previous database write is complete */
.take_until(future::join(sleep(delay), do_write_.clone()))
.boxed();
/* Read up until an error is encountered,
* handle log lines first and then the error... */
let mut joined = String::new();
while let Some(line) = read_lines.next().await {
match line {
Ok(line) => {
if line.is_empty() {
continue;
}
append_with_limit(&mut joined, &line, &mut log_remaining);
if log_remaining == 0 {
tracing::info!(%job_id, "Too many logs lines for job {job_id}");
let _ = set_too_many_logs.send(true);
joined.push_str(&format!(
"Job logs or result reached character limit of {MAX_RESULT_SIZE}; killing job."
));
/* stop reading and drop our streams fairly quickly */
break;
}
}
Err(err) => {
result = Err(err);
break;
}
}
}
/* Ensure the last flush completed before starting a new one.
*
* This shouldn't pause since `take_until()` reads lines until `do_write`
* resolves. We only stop reading lines before `take_until()` resolves if we reach
* EOF or a read error. In those cases, waiting on a database query to complete is
* fine because we're done. */
if let Some(Ok(p)) = do_write_
.then(|()| write_result)
.await
.err()
.map(|err| err.try_into_panic())
{
panic::resume_unwind(p);
}
let joined_len = joined.len() as u64;
log_total_size += joined_len;
let compact_logs = log_total_size > LARGE_LOG_THRESHOLD_SIZE as u64;
if compact_logs {
log_total_size = 0;
}
let worker_name = worker.to_string();
let w_id2 = w_id.to_string();
(do_write, write_result) = tokio::spawn(append_job_logs(job_id, w_id2, joined, db.clone(), compact_logs, pg_log_total_size.clone(), worker_name)).remote_handle();
if let Err(err) = result {
tracing::error!(%job_id, %err, "error reading output for job {job_id} '{child_name}': {err}");
break;
}
if *set_too_many_logs.borrow() {
break;
}
}
/* drop our end of the pipe */
drop(output);
if let Some(Ok(p)) = do_write
.then(|()| write_result)
.await
.err()
.map(|err| err.try_into_panic())
{
panic::resume_unwind(p);
}
}.instrument(trace_span!("child_lines"));
let (wait_result, _) = tokio::join!(wait_on_child, lines);
let success = wait_result.is_ok()
&& wait_result.as_ref().unwrap().is_ok()
&& wait_result.as_ref().unwrap().as_ref().unwrap().success();
tracing::info!(%job_id, %success, %mem_peak, %worker, "child process '{child_name}' took {}ms", start.elapsed().as_millis());
match wait_result {
_ if *too_many_logs.borrow() => Err(Error::ExecutionErr(format!(
"logs or result reached limit. (current max size: {MAX_RESULT_SIZE} characters)"
))),
Ok(Ok(status)) => process_status(status),
Ok(Err(kill_reason)) => match kill_reason {
KillReason::AlreadyCompleted => {
Err(Error::AlreadyCompleted("Job already completed".to_string()))
}
_ => Err(Error::ExecutionErr(format!(
"job process killed because {kill_reason:#?}"
))),
},
Err(err) => Err(Error::ExecutionErr(format!("job process io error: {err}"))),
}
}
async fn get_mem_peak(pid: Option<u32>, nsjail: bool) -> i32 {
if pid.is_none() {
return -1;
}
let pid = if nsjail {
// This is a bit hacky, but the process id of the nsjail process is the pid of nsjail + 1.
// Ideally, we would get the number from fork() itself. This works in MOST cases.
pid.unwrap() + 1
} else {
pid.unwrap()
};
if let Ok(file) = File::open(format!("/proc/{}/status", pid)).await {
let mut lines = BufReader::new(file).lines();
while let Some(line) = lines.next_line().await.unwrap_or(None) {
if line.starts_with("VmHWM:") {
return line
.split_whitespace()
.nth(1)
.and_then(|s| s.parse::<i32>().ok())
.unwrap_or(-1);
};
}
-2
} else {
// rand::random::<i32>() % 100 // to remove - used to fake memory data on MacOS
-3
}
}
pub async fn run_future_with_polling_update_job_poller<Fut, T>(
job_id: Uuid,
timeout: Option<i32>,
db: &DB,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
result_f: Fut,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<T>
where
Fut: Future<Output = anyhow::Result<T>>,
{
let (tx, rx) = broadcast::channel::<()>(3);
let update_job = update_job_poller(
job_id,
db,
mem_peak,
canceled_by_ref,
|| async { 0 },
worker_name,
w_id,
rx,
occupancy_metrics,
);
let timeout_ms = u64::try_from(
resolve_job_timeout(&db, &w_id, job_id, timeout)
.await
.0
.as_millis(),
)
.unwrap_or(200000);
let rows = tokio::select! {
biased;
result = tokio::time::timeout(std::time::Duration::from_millis(timeout_ms), result_f) => result
.map_err(|e| {
tracing::error!("Query timeout: {}", e);
Error::ExecutionErr(format!("Query timeout after (>{}s)", timeout_ms/1000))
})?,
ex = update_job, if job_id != Uuid::nil() => {
match ex {
UpdateJobPollingExit::Done => Err(Error::ExecutionErr("Job cancelled".to_string())).map_err(to_anyhow)?,
UpdateJobPollingExit::AlreadyCompleted => Err(Error::AlreadyCompleted("Job already completed".to_string())).map_err(to_anyhow)?,
}
}
}?;
drop(tx);
Ok(rows)
}
pub enum UpdateJobPollingExit {
Done,
AlreadyCompleted,
}
pub async fn update_job_poller<F, Fut>(
job_id: Uuid,
db: &DB,
mem_peak: &mut i32,
canceled_by_ref: &mut Option<CanceledBy>,
get_mem: F,
worker_name: &str,
w_id: &str,
mut rx: broadcast::Receiver<()>,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> UpdateJobPollingExit
where
F: Fn() -> Fut,
Fut: Future<Output = i32>,
{
let update_job_interval = Duration::from_millis(500);
let db = db.clone();
let mut interval = interval(update_job_interval);
interval.set_missed_tick_behavior(MissedTickBehavior::Skip);
let mut i = 0;
#[cfg(feature = "enterprise")]
let mut memory_metric_id: Result<String, Error> =
Err(Error::NotFound("not yet initialized".to_string()));
loop {
tokio::select!(
_ = rx.recv() => break,
_ = interval.tick() => {
// update the last_ping column every 5 seconds
i+=1;
if i == 1 || i % 10 == 0 {
let memory_usage = get_worker_memory_usage();
let wm_memory_usage = get_windmill_memory_usage();
tracing::info!("job {job_id} on {worker_name} in {w_id} worker memory snapshot {}kB/{}kB", memory_usage.unwrap_or_default()/1024, wm_memory_usage.unwrap_or_default()/1024);
let occupancy = occupancy_metrics.as_mut().map(|x| x.update_occupancy_metrics());
if job_id != Uuid::nil() {
sqlx::query!(
"UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4,
occupancy_rate = $6, occupancy_rate_15s = $7, occupancy_rate_5m = $8, occupancy_rate_30m = $9 WHERE worker = $5",
&job_id,
&w_id,
memory_usage,
wm_memory_usage,
&worker_name,
occupancy.map(|x| x.0),
occupancy.and_then(|x| x.1),
occupancy.and_then(|x| x.2),
occupancy.and_then(|x| x.3),
)
.execute(&db)
.await
.expect("update worker ping");
}
}
let current_mem = get_mem().await;
if current_mem > *mem_peak {
*mem_peak = current_mem
}
tracing::info!("job {job_id} on {worker_name} in {w_id} still running. mem: {current_mem}kB, peak mem: {mem_peak}kB");
let update_job_row = i == 2 || (!*SLOW_LOGS && (i < 20 || (i < 120 && i % 5 == 0) || i % 10 == 0)) || i % 20 == 0;
if update_job_row {
#[cfg(feature = "enterprise")]
{
if job_id != Uuid::nil() {
// tracking metric starting at i >= 2 b/c first point it useless and we don't want to track metric for super fast jobs
if i == 2 {
memory_metric_id = job_metrics::register_metric_for_job(
&db,
w_id.to_string(),
job_id,
"memory_kb".to_string(),
job_metrics::MetricKind::TimeseriesInt,
Some("Job Memory Footprint (kB)".to_string()),
)
.await;
}
if let Ok(ref metric_id) = memory_metric_id {
if let Err(err) = job_metrics::record_metric(&db, w_id.to_string(), job_id, metric_id.to_owned(), job_metrics::MetricNumericValue::Integer(current_mem)).await {
tracing::error!("Unable to save memory stat for job {} in workspace {}. Error was: {:?}", job_id, w_id, err);
}
}
}
}
if job_id != Uuid::nil() {
let (canceled, canceled_by, canceled_reason, already_completed) = sqlx::query_as::<_, (bool, Option<String>, Option<String>, bool)>("UPDATE queue SET mem_peak = $1, last_ping = now() WHERE id = $2 RETURNING canceled, canceled_by, canceled_reason, false")
.bind(*mem_peak)
.bind(job_id)
.fetch_optional(&db)
.await
.unwrap_or_else(|e| {
tracing::error!(%e, "error updating job {job_id}: {e:#}");
Some((false, None, None, false))
})
.unwrap_or_else(|| {
// if the job is not in queue, it can only be in the completed_job so it is already complete
(false, None, None, true)
});
if already_completed {
return UpdateJobPollingExit::AlreadyCompleted
}
if canceled {
canceled_by_ref.replace(CanceledBy {
username: canceled_by.clone(),
reason: canceled_reason.clone(),
});
break
}
}
}
},
);
}
tracing::info!("job {job_id} finished");
UpdateJobPollingExit::Done
}
/// takes stdout and stderr from Child, panics if either are not present
///
/// builds a stream joining both stdout and stderr each read line by line
fn child_joined_output_stream(
child: &mut Child,
) -> impl stream::FusedStream<Item = io::Result<String>> {
let stderr = child
.stderr
.take()
.expect("child did not have a handle to stdout");
let stdout = child
.stdout
.take()
.expect("child did not have a handle to stdout");
let stdout = BufReader::new(stdout).lines();
let stderr = BufReader::new(stderr).lines();
stream::select(lines_to_stream(stderr), lines_to_stream(stdout))
}
pub fn lines_to_stream<R: tokio::io::AsyncBufRead + Unpin>(
mut lines: tokio::io::Lines<R>,
) -> impl futures::Stream<Item = io::Result<String>> {
stream::poll_fn(move |cx| {
std::pin::Pin::new(&mut lines)
.poll_next_line(cx)
.map(|result| result.transpose())
})
}
pub fn process_status(status: ExitStatus) -> error::Result<()> {
if status.success() {
Ok(())
} else if let Some(code) = status.code() {
Err(error::Error::ExitStatus(code))
} else {
#[cfg(any(target_os = "linux", target_os = "macos"))]
return Err(error::Error::ExecutionErr(format!(
"process terminated by signal: {:#?}, stopped_signal: {:#?}, core_dumped: {}",
status.signal(),
status.stopped_signal(),
status.core_dumped()
)));
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
return Err(error::Error::ExecutionErr(String::from(
"process terminated by signal",
)));
}
}
backend/windmill-worker/src/job_logger.rs
+285
View File
@@ -0,0 +1,285 @@
use deno_ast::swc::parser::lexer::util::CharExt;
use itertools::Itertools;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
use object_store::path::Path;
use regex::Regex;
#[cfg(all(feature = "enterprise", feature = "parquet"))]
use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
use windmill_common::error::{self};
use windmill_common::worker::{CLOUD_HOSTED, TMP_DIR};
use windmill_queue::append_logs;
use std::sync::atomic::AtomicU32;
use std::sync::Arc;
use uuid::Uuid;
use windmill_common::DB;
pub enum CompactLogs {
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
NotEE,
#[allow(dead_code)]
NoS3,
#[allow(dead_code)]
S3,
}
async fn compact_logs(
job_id: Uuid,
w_id: &str,
db: &DB,
nlogs: String,
total_size: Arc<AtomicU32>,
compact_kind: CompactLogs,
_worker_name: &str,
) -> error::Result<(String, String)> {
let mut prev_logs = sqlx::query_scalar!(
"SELECT logs FROM job_logs WHERE job_id = $1 AND workspace_id = $2",
job_id,
w_id
)
.fetch_optional(db)
.await?
.flatten()
.unwrap_or_default();
let size = prev_logs.char_indices().count() as i32;
let nlogs_len = nlogs.char_indices().count();
let to_keep_in_db = usize::max(
usize::min(nlogs_len, 3000),
nlogs_len % LARGE_LOG_THRESHOLD_SIZE,
);
let extra_split = to_keep_in_db < nlogs_len;
let stored_in_storage_len = if extra_split {
nlogs_len - to_keep_in_db
} else {
0
};
let extra_to_newline = nlogs
.chars()
.skip(stored_in_storage_len)
.find_position(|x| x.is_line_break())
.map(|(i, _)| i)
.unwrap_or(to_keep_in_db);
let stored_in_storage_to_newline = stored_in_storage_len + extra_to_newline;
let (append_to_storage, stored_in_db) = if extra_split {
if stored_in_storage_to_newline == nlogs.len() {
(nlogs.as_ref(), "".to_string())
} else {
let split_idx = nlogs
.char_indices()
.nth(stored_in_storage_to_newline)
.map(|(i, _)| i)
.unwrap_or(0);
let (append_to_storage, stored_in_db) = nlogs.split_at(split_idx);
// tracing::error!("{append_to_storage} ||||| {stored_in_db}");
// tracing::error!(
// "{:?} {:?} {} {}",
// excess_prev_logs.lines().last(),
// current_logs.lines().next(),
// split_idx,
// excess_size_modulo
// );
(append_to_storage, stored_in_db.to_string())
}
} else {
// tracing::error!("{:?}", nlogs.lines().last());
("", nlogs.to_string())
};
let new_size_with_excess = size + stored_in_storage_to_newline as i32;
let new_size = total_size.fetch_add(
new_size_with_excess as u32,
std::sync::atomic::Ordering::SeqCst,
) + new_size_with_excess as u32;
let path = format!(
"logs/{job_id}/{}_{new_size}.txt",
chrono::Utc::now().timestamp_millis()
);
let mut new_current_logs = match compact_kind {
CompactLogs::NoS3 => format!("\n[windmill] No object storage set in instance settings. Previous logs have been saved to disk at {path}"),
CompactLogs::S3 => format!("\n[windmill] Previous logs have been saved to object storage at {path}"),
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
CompactLogs::NotEE => format!("\n[windmill] Previous logs have been saved to disk at {path}"),
};
new_current_logs.push_str(&stored_in_db);
sqlx::query!(
"UPDATE job_logs SET logs = $1, log_offset = $2,
log_file_index = array_append(coalesce(log_file_index, array[]::text[]), $3)
WHERE workspace_id = $4 AND job_id = $5",
new_current_logs,
new_size as i32,
path,
w_id,
job_id
)
.execute(db)
.await?;
prev_logs.push_str(&append_to_storage);
return Ok((prev_logs, path));
}
async fn default_disk_log_storage(
job_id: Uuid,
w_id: &str,
db: &DB,
nlogs: String,
total_size: Arc<AtomicU32>,
compact_kind: CompactLogs,
worker_name: &str,
) {
match compact_logs(
job_id,
&w_id,
&db,
nlogs,
total_size,
compact_kind,
worker_name,
)
.await
{
Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
Ok((prev_logs, path)) => {
let path = format!("{}/{}", TMP_DIR, path);
let splitted = &path.split("/").collect_vec();
tokio::fs::create_dir_all(splitted.into_iter().take(splitted.len() - 1).join("/"))
.await
.map_err(|e| {
tracing::error!("Could not create logs directory: {e:?}",);
e
})
.ok();
let created = tokio::fs::File::create(&path).await;
if let Err(e) = created {
tracing::error!("Could not create logs file {path}: {e:?}",);
return;
}
if let Err(e) = tokio::fs::write(&path, prev_logs).await {
tracing::error!("Could not write to logs file {path}: {e:?}");
} else {
tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to disk at {path}");
}
}
}
}
pub(crate) async fn append_job_logs(
job_id: Uuid,
w_id: String,
logs: String,
db: DB,
must_compact_logs: bool,
total_size: Arc<AtomicU32>,
worker_name: String,
) -> () {
if must_compact_logs {
#[cfg(all(feature = "enterprise", feature = "parquet"))]
if let Some(os) = OBJECT_STORE_CACHE_SETTINGS.read().await.clone() {
match compact_logs(
job_id,
&w_id,
&db,
logs,
total_size,
CompactLogs::S3,
&worker_name,
)
.await
{
Err(e) => tracing::error!("Could not compact logs for job {job_id}: {e:?}",),
Ok((prev_logs, path)) => {
tracing::info!("Logs length of {job_id} has exceeded a threshold. Previous logs have been saved to object storage at {path}");
let path2 = path.clone();
if let Err(e) = os
.put(&Path::from(path), prev_logs.to_string().into_bytes().into())
.await
{
tracing::error!("Could not save logs to s3: {e:?}");
}
tracing::info!("Logs of {job_id} saved to object storage at {path2}");
}
}
} else {
default_disk_log_storage(
job_id,
&w_id,
&db,
logs,
total_size,
CompactLogs::NoS3,
&worker_name,
)
.await;
}
#[cfg(not(all(feature = "enterprise", feature = "parquet")))]
{
default_disk_log_storage(
job_id,
&w_id,
&db,
logs,
total_size,
CompactLogs::NotEE,
&worker_name,
)
.await;
}
} else {
append_logs(&job_id, w_id, logs, db).await;
}
}
pub const LARGE_LOG_THRESHOLD_SIZE: usize = 9000;
lazy_static::lazy_static! {
static ref RE_00: Regex = Regex::new('\u{00}'.to_string().as_str()).unwrap();
pub static ref NO_LOGS_AT_ALL: bool = std::env::var("NO_LOGS_AT_ALL").ok().is_some_and(|x| x == "1" || x == "true");
}
// as a detail, `BufReader::lines()` removes \n and \r\n from the strings it yields,
// so this pushes \n to thd destination string in each call
pub fn append_with_limit(dst: &mut String, src: &str, limit: &mut usize) {
if *NO_LOGS_AT_ALL {
return;
}
let src_str;
let src = {
src_str = RE_00.replace_all(src, "");
src_str.as_ref()
};
if !*CLOUD_HOSTED {
dst.push('\n');
dst.push_str(&src);
return;
} else {
if *limit > 0 {
dst.push('\n');
}
*limit -= 1;
}
let src_len = src.chars().count();
if src_len <= *limit {
dst.push_str(&src);
*limit -= src_len;
} else {
let byte_pos = src
.char_indices()
.skip(*limit)
.next()
.map(|(byte_pos, _)| byte_pos)
.unwrap_or(0);
dst.push_str(&src[0..byte_pos]);
*limit = 0;
}
}
backend/windmill-worker/src/js_eval.rs
+4 -1
View File
@@ -40,7 +40,8 @@ use windmill_common::{error::Error, flow_status::JobResult, DB};
use windmill_queue::CanceledBy;
use crate::{
common::{run_future_with_polling_update_job_poller, unsafe_raw},
common::{unsafe_raw, OccupancyMetrics},
handle_child::run_future_with_polling_update_job_poller,
AuthedClient,
};
@@ -688,6 +689,7 @@ pub async fn eval_fetch_timeout(
worker_name: &str,
w_id: &str,
load_client: bool,
occupation_metrics: &mut OccupancyMetrics,
) -> anyhow::Result<(Box<RawValue>, String)> {
let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
@@ -836,6 +838,7 @@ pub async fn eval_fetch_timeout(
async { result_f.await? },
worker_name,
w_id,
&mut Some(occupation_metrics),
)
.await
.map_err(|e| {
backend/windmill-worker/src/lib.rs
+5 -1
View File
@@ -5,7 +5,9 @@ mod mssql_executor;
#[cfg(feature = "enterprise")]
mod snowflake_executor;
mod ansible_executor;
mod bash_executor;
mod bun_executor;
pub mod common;
mod config;
@@ -15,6 +17,8 @@ mod deno_executor;
mod global_cache;
mod go_executor;
mod graphql_executor;
mod handle_child;
mod job_logger;
mod js_eval;
mod mysql_executor;
mod pg_executor;
@@ -25,7 +29,7 @@ mod rust_executor;
mod worker;
mod worker_flow;
mod worker_lockfiles;
mod ansible_executor;
pub use worker::*;
pub use result_processor::handle_job_error;
backend/windmill-worker/src/mssql_executor.rs
+4 -1
View File
@@ -14,7 +14,8 @@ use windmill_common::{error::to_anyhow, jobs::QueuedJob};
use windmill_parser_sql::{parse_db_resource, parse_mssql_sig};
use windmill_queue::{append_logs, CanceledBy};
use crate::common::{build_args_values, run_future_with_polling_update_job_poller};
use crate::common::{build_args_values, OccupancyMetrics};
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::AuthedClientBackgroundTask;
#[derive(Deserialize)]
@@ -39,6 +40,7 @@ pub async fn do_mssql(
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<Box<RawValue>> {
let mssql_args = build_args_values(job, client, db).await?;
@@ -157,6 +159,7 @@ pub async fn do_mssql(
result_f,
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
)
.await?;
backend/windmill-worker/src/mysql_executor.rs
+4 -1
View File
@@ -22,7 +22,8 @@ use windmill_parser_sql::{
use windmill_queue::CanceledBy;
use crate::{
common::{build_args_map, run_future_with_polling_update_job_poller},
common::{build_args_map, OccupancyMetrics},
handle_child::run_future_with_polling_update_job_poller,
AuthedClientBackgroundTask,
};
@@ -110,6 +111,7 @@ pub async fn do_mysql(
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
column_order: &mut Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> windmill_common::error::Result<Box<RawValue>> {
let args = build_args_map(job, client, db).await?.map(Json);
let job_args = if args.is_some() {
@@ -295,6 +297,7 @@ pub async fn do_mysql(
result_f,
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
)
.await?;
backend/windmill-worker/src/pg_executor.rs
+4 -1
View File
@@ -38,7 +38,8 @@ use windmill_parser_sql::{
};
use windmill_queue::CanceledBy;
use crate::common::{build_args_values, run_future_with_polling_update_job_poller, sizeof_val};
use crate::common::{build_args_values, sizeof_val, OccupancyMetrics};
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::{AuthedClientBackgroundTask, MAX_RESULT_SIZE};
use bytes::{Buf, BytesMut};
use lazy_static::lazy_static;
@@ -162,6 +163,7 @@ pub async fn do_postgresql(
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
column_order: &mut Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<Box<RawValue>> {
let pg_args = build_args_values(job, client, db).await?;
@@ -351,6 +353,7 @@ pub async fn do_postgresql(
result_f,
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
)
.await?;
backend/windmill-worker/src/php_executor.rs
+8 -2
View File
@@ -15,9 +15,10 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
read_result, start_child_process,
create_args_and_out_file, get_main_override, get_reserved_variables, read_result,
start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
AuthedClientBackgroundTask, COMPOSER_CACHE_DIR, COMPOSER_PATH, DISABLE_NSJAIL, DISABLE_NUSER,
NSJAIL_PATH, PHP_PATH,
};
@@ -70,6 +71,7 @@ pub async fn composer_install(
worker_name: &str,
requirements: String,
lock: Option<String>,
occupancy_metrics: &mut OccupancyMetrics,
) -> Result<String> {
check_php_exists()?;
@@ -101,6 +103,7 @@ pub async fn composer_install(
"composer install",
None,
false,
&mut Some(occupancy_metrics),
)
.await?;
@@ -160,6 +163,7 @@ pub async fn handle_php_job(
worker_name: &str,
envs: HashMap<String, String>,
shared_mount: &str,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<Box<RawValue>> {
check_php_exists()?;
@@ -190,6 +194,7 @@ pub async fn handle_php_job(
worker_name,
composer_json,
composer_lock,
occupancy_metrics,
)
.await?;
"require './vendor/autoload.php';"
@@ -346,6 +351,7 @@ try {{
"php run",
job.timeout,
false,
&mut Some(occupancy_metrics),
)
.await?;
read_result(job_dir).await
backend/windmill-worker/src/python_executor.rs
+14 -2
View File
@@ -55,9 +55,10 @@ use windmill_common::s3_helpers::OBJECT_STORE_CACHE_SETTINGS;
use crate::{
common::{
create_args_and_out_file, get_main_override, get_reserved_variables, handle_child,
read_file, read_result, start_child_process,
create_args_and_out_file, get_main_override, get_reserved_variables, read_file,
read_result, start_child_process, OccupancyMetrics,
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY,
LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL,
PIP_INDEX_URL, TZ_ENV,
@@ -101,6 +102,7 @@ pub async fn pip_compile(
db: &Pool<Postgres>,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<String> {
let mut logs = String::new();
logs.push_str(&format!("\nresolving dependencies..."));
@@ -210,6 +212,7 @@ pub async fn pip_compile(
"pip-compile",
None,
false,
occupancy_metrics,
)
.await
.map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?;
@@ -247,6 +250,7 @@ pub async fn handle_python_job(
base_internal_url: &str,
envs: HashMap<String, String>,
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> windmill_common::error::Result<Box<RawValue>> {
let script_path = crate::common::use_flow_root_path(job.script_path());
let additional_python_paths = handle_python_deps(
@@ -261,6 +265,7 @@ pub async fn handle_python_job(
worker_dir,
mem_peak,
canceled_by,
&mut Some(occupancy_metrics),
)
.await?;
@@ -485,6 +490,7 @@ mount {{
"python run",
job.timeout,
false,
&mut Some(occupancy_metrics),
)
.await?;
@@ -753,6 +759,7 @@ async fn handle_python_deps(
worker_dir: &str,
mem_peak: &mut i32,
canceled_by: &mut Option<CanceledBy>,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<Vec<String>> {
create_dependencies_dir(job_dir).await;
@@ -790,6 +797,7 @@ async fn handle_python_deps(
db,
worker_name,
w_id,
occupancy_metrics,
)
.await
.map_err(|e| {
@@ -813,6 +821,7 @@ async fn handle_python_deps(
worker_name,
job_dir,
worker_dir,
occupancy_metrics,
)
.await?;
additional_python_paths.append(&mut venv_path);
@@ -834,6 +843,7 @@ pub async fn handle_python_reqs(
worker_name: &str,
job_dir: &str,
worker_dir: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> error::Result<Vec<String>> {
let mut req_paths: Vec<String> = vec![];
let mut vars = vec![("PATH", PATH_ENV.as_str())];
@@ -1089,6 +1099,7 @@ pub async fn handle_python_reqs(
&format!("pip install {req}"),
None,
false,
occupancy_metrics,
)
.await;
tracing::info!(
@@ -1173,6 +1184,7 @@ pub async fn start_worker(
job_dir,
&mut mem_peak,
&mut canceled_by,
&mut None,
)
.await?;
backend/windmill-worker/src/result_processor.rs
+185 -23
View File
@@ -1,16 +1,26 @@
use serde::Serialize;
use sqlx::{types::Json, Pool, Postgres};
use std::{collections::HashMap, sync::Arc};
use std::{
collections::HashMap,
sync::{
atomic::{AtomicBool, AtomicU16, Ordering},
Arc,
},
};
use uuid::Uuid;
use windmill_common::{
add_time,
error::{self, Error},
jobs::QueuedJob,
worker::to_raw_value,
jobs::{JobKind, QueuedJob},
worker::{to_raw_value, WORKER_GROUP},
DB,
};
#[cfg(feature = "benchmark")]
use windmill_common::bench::{BenchmarkInfo, BenchmarkIter};
use windmill_queue::{append_logs, get_queued_job, CanceledBy, WrappedError};
#[cfg(feature = "prometheus")]
@@ -18,7 +28,13 @@ use windmill_queue::register_metric;
use serde_json::{json, value::RawValue};
use tokio::sync::mpsc::Sender;
use tokio::{
sync::{
self,
mpsc::{Receiver, Sender},
},
task::JoinHandle,
};
use windmill_queue::{add_completed_job, add_completed_job_error};
@@ -26,9 +42,149 @@ use crate::{
bash_executor::ANSI_ESCAPE_RE,
common::{read_result, save_in_cache},
worker_flow::update_flow_status_after_job_completion,
AuthedClient, Histo, JobCompleted, JobCompletedSender, SameWorkerSender, SendResult,
AuthedClient, JobCompleted, JobCompletedSender, SameWorkerSender, SendResult, INIT_SCRIPT_TAG,
};
pub fn start_background_processor<R>(
mut job_completed_rx: Receiver<SendResult>,
job_completed_sender: Sender<SendResult>,
same_worker_queue_size: Arc<AtomicU16>,
job_completed_processor_is_done: Arc<AtomicBool>,
base_internal_url: String,
db: DB,
worker_dir: String,
same_worker_tx: SameWorkerSender,
rsmq: Option<R>,
worker_name: String,
killpill_tx: sync::broadcast::Sender<()>,
is_dedicated_worker: bool,
) -> JoinHandle<()>
where
R: rsmq_async::RsmqConnection + Send + Sync + Clone + 'static,
{
tokio::spawn(async move {
let mut has_been_killed = false;
#[cfg(feature = "benchmark")]
let mut infos = BenchmarkInfo::new();
//if we have been killed, we want to drain the queue of jobs
while let Some(sr) = {
if has_been_killed && same_worker_queue_size.load(Ordering::SeqCst) == 0 {
job_completed_rx.try_recv().ok()
} else {
job_completed_rx.recv().await
}
} {
#[cfg(feature = "benchmark")]
let mut bench = BenchmarkIter::new();
match sr {
SendResult::JobCompleted(jc) => {
let rsmq = rsmq.clone();
let is_init_script_and_failure =
!jc.success && jc.job.tag.as_str() == INIT_SCRIPT_TAG;
let is_dependency_job = matches!(
jc.job.job_kind,
JobKind::Dependencies | JobKind::FlowDependencies
);
handle_receive_completed_job(
jc,
&base_internal_url,
&db,
&worker_dir,
&same_worker_tx,
rsmq,
&worker_name,
job_completed_sender.clone(),
#[cfg(feature = "benchmark")]
&mut bench,
)
.await;
if is_init_script_and_failure {
tracing::error!("init script errored, exiting");
killpill_tx.send(()).unwrap_or_default();
break;
}
if is_dependency_job && is_dedicated_worker {
tracing::error!("Dedicated worker executed a dependency job, a new script has been deployed. Exiting expecting to be restarted.");
sqlx::query!(
"UPDATE config SET config = config WHERE name = $1",
format!("worker__{}", *WORKER_GROUP)
)
.execute(&db)
.await
.expect("update config to trigger restart of all dedicated workers at that config");
killpill_tx.send(()).unwrap_or_default();
}
add_time!(bench, "job completed processed");
#[cfg(feature = "benchmark")]
{
infos.add_iter(bench, true);
}
}
SendResult::UpdateFlow {
flow,
w_id,
success,
result,
worker_dir,
stop_early_override,
token,
} => {
// let r;
tracing::info!(parent_flow = %flow, "updating flow status");
if let Err(e) = update_flow_status_after_job_completion(
&db,
&AuthedClient {
base_internal_url: base_internal_url.to_string(),
workspace: w_id.clone(),
token: token.clone(),
force_client: None,
},
flow,
&Uuid::nil(),
&w_id,
success,
Arc::new(result),
true,
same_worker_tx.clone(),
&worker_dir,
stop_early_override,
rsmq.clone(),
&worker_name,
job_completed_sender.clone(),
#[cfg(feature = "benchmark")]
&mut bench,
)
.await
{
tracing::error!("Error updating flow status after job completion for {flow} on {worker_name}: {e:#}");
}
}
SendResult::Kill => {
has_been_killed = true;
}
}
}
job_completed_processor_is_done.store(true, Ordering::SeqCst);
tracing::info!("finished processing all completed jobs");
#[cfg(feature = "benchmark")]
{
infos
.write_to_file("profiling_result_processor.json")
.expect("write to file profiling");
}
})
}
async fn send_job_completed(
job_completed_tx: JobCompletedSender,
job: Arc<QueuedJob>,
@@ -168,9 +324,8 @@ pub async fn handle_receive_completed_job<
same_worker_tx: &SameWorkerSender,
rsmq: Option<R>,
worker_name: &str,
worker_save_completed_job_duration: Option<Histo>,
worker_flow_transition_duration: Option<Histo>,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) {
let token = jc.token.clone();
let workspace = jc.job.workspace_id.clone();
@@ -191,9 +346,9 @@ pub async fn handle_receive_completed_job<
same_worker_tx.clone(),
rsmq.clone(),
worker_name,
worker_save_completed_job_duration,
worker_flow_transition_duration,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
bench,
)
.await
{
@@ -210,6 +365,8 @@ pub async fn handle_receive_completed_job<
rsmq.clone(),
worker_name,
job_completed_tx,
#[cfg(feature = "benchmark")]
bench,
)
.await;
}
@@ -224,9 +381,8 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
same_worker_tx: SameWorkerSender,
rsmq: Option<R>,
worker_name: &str,
_worker_save_completed_job_duration: Option<Histo>,
_worker_flow_transition_duration: Option<Histo>,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> windmill_common::error::Result<()> {
if success {
// println!("bef completed job{:?}", SystemTime::now());
@@ -238,10 +394,7 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
let parent_job = job.parent_job.clone();
let job_id = job.id.clone();
let workspace_id = job.workspace_id.clone();
#[cfg(feature = "prometheus")]
let timer = _worker_save_completed_job_duration
.as_ref()
.map(|x| x.start_timer());
add_completed_job(
db,
&job,
@@ -252,19 +405,16 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
canceled_by,
rsmq.clone(),
false,
#[cfg(feature = "benchmark")]
bench,
)
.await?;
drop(job);
#[cfg(feature = "prometheus")]
timer.map(|x| x.stop_and_record());
add_time!(bench, "add_completed_job END");
if is_flow_step {
if let Some(parent_job) = parent_job {
#[cfg(feature = "prometheus")]
let timer = _worker_flow_transition_duration
.as_ref()
.map(|x| x.start_timer());
tracing::info!(parent_flow = %parent_job, subflow = %job_id, "updating flow status (2)");
update_flow_status_after_job_completion(
db,
@@ -281,12 +431,13 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
rsmq.clone(),
worker_name,
job_completed_tx,
#[cfg(feature = "benchmark")]
bench,
)
.await?;
#[cfg(feature = "prometheus")]
timer.map(|x| x.stop_and_record());
}
}
add_time!(bench, "updated flow status END");
} else {
let result = add_completed_job_error(
db,
@@ -299,6 +450,8 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
rsmq.clone(),
worker_name,
false,
#[cfg(feature = "benchmark")]
bench,
)
.await?;
if job.is_flow_step {
@@ -319,6 +472,8 @@ pub async fn process_completed_job<R: rsmq_async::RsmqConnection + Send + Sync +
rsmq,
worker_name,
job_completed_tx,
#[cfg(feature = "benchmark")]
bench,
)
.await?;
}
@@ -341,6 +496,7 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
rsmq: Option<R>,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) {
let err = match err {
Error::JsonErr(err) => err,
@@ -365,6 +521,8 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
rsmq_2,
worker_name,
false,
#[cfg(feature = "benchmark")]
bench,
)
.await
};
@@ -399,6 +557,8 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
rsmq.clone(),
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
bench,
)
.await;
@@ -424,6 +584,8 @@ pub async fn handle_job_error<R: rsmq_async::RsmqConnection + Send + Sync + Clon
rsmq,
worker_name,
false,
#[cfg(feature = "benchmark")]
bench,
)
.await;
}
backend/windmill-worker/src/rust_executor.rs
+10 -2
View File
@@ -15,9 +15,10 @@ use windmill_queue::{append_logs, CanceledBy};
use crate::{
common::{
create_args_and_out_file, get_reserved_variables, handle_child, read_result,
start_child_process,
create_args_and_out_file, get_reserved_variables, read_result, start_child_process,
OccupancyMetrics,
},
handle_child::handle_child,
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
RUST_CACHE_DIR, TZ_ENV,
};
@@ -113,6 +114,7 @@ pub async fn generate_cargo_lockfile(
db: &sqlx::Pool<sqlx::Postgres>,
worker_name: &str,
w_id: &str,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<String> {
check_cargo_exists()?;
@@ -137,6 +139,7 @@ pub async fn generate_cargo_lockfile(
"cargo generate-lockfile",
None,
false,
&mut Some(occupancy_metrics),
)
.await?;
@@ -157,6 +160,7 @@ pub async fn build_rust_crate(
w_id: &str,
base_internal_url: &str,
hash: &str,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<String> {
let bin_path = format!("{}/{hash}", RUST_CACHE_DIR);
@@ -185,6 +189,7 @@ pub async fn build_rust_crate(
"rust build",
None,
false,
&mut Some(occupancy_metrics),
)
.await?;
append_logs(job_id, w_id, "\n\n", db).await;
@@ -262,6 +267,7 @@ pub async fn handle_rust_job(
base_internal_url: &str,
worker_name: &str,
envs: HashMap<String, String>,
occupancy_metrics: &mut OccupancyMetrics,
) -> Result<Box<RawValue>, Error> {
check_cargo_exists()?;
@@ -305,6 +311,7 @@ pub async fn handle_rust_job(
&job.workspace_id,
base_internal_url,
&hash,
occupancy_metrics,
)
.await?
};
@@ -367,6 +374,7 @@ pub async fn handle_rust_job(
"rust run",
job.timeout,
false,
&mut Some(occupancy_metrics),
)
.await?;
read_result(job_dir).await
backend/windmill-worker/src/snowflake_executor.rs
+4 -1
View File
@@ -18,7 +18,8 @@ use windmill_queue::{CanceledBy, HTTP_CLIENT};
use serde::{Deserialize, Serialize};
use crate::common::{resolve_job_timeout, run_future_with_polling_update_job_poller};
use crate::common::{resolve_job_timeout, OccupancyMetrics};
use crate::handle_child::run_future_with_polling_update_job_poller;
use crate::{common::build_args_values, AuthedClientBackgroundTask};
#[derive(Serialize)]
@@ -237,6 +238,7 @@ pub async fn do_snowflake(
canceled_by: &mut Option<CanceledBy>,
worker_name: &str,
column_order: &mut Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> windmill_common::error::Result<Box<RawValue>> {
let snowflake_args = build_args_values(job, client, db).await?;
@@ -383,6 +385,7 @@ pub async fn do_snowflake(
result_f.map_err(to_anyhow),
worker_name,
&job.workspace_id,
&mut Some(occupancy_metrics),
)
.await?;
*mem_peak = (r.get().len() / 1000) as i32;
backend/windmill-worker/src/worker.rs
+240 -544
View File
File diff suppressed because it is too large Load Diff
backend/windmill-worker/src/worker_flow.rs
+202 -169
View File
@@ -29,7 +29,10 @@ use sqlx::FromRow;
use tokio::sync::mpsc::Sender;
use tracing::instrument;
use uuid::Uuid;
use windmill_common::add_time;
use windmill_common::auth::JobPerms;
#[cfg(feature = "benchmark")]
use windmill_common::bench::BenchmarkIter;
use windmill_common::db::Authed;
use windmill_common::flow_status::{
ApprovalConditions, FlowStatusModuleWParent, Iterator, JobResult,
@@ -50,8 +53,8 @@ use windmill_common::{
};
use windmill_queue::schedule::get_schedule_opt;
use windmill_queue::{
add_completed_job, add_completed_job_error, append_logs, get_queued_job,
handle_maybe_scheduled_job, CanceledBy, PushArgs, PushIsolationLevel, WrappedError,
add_completed_job, add_completed_job_error, append_logs, handle_maybe_scheduled_job,
CanceledBy, PushArgs, PushIsolationLevel, WrappedError,
};
type DB = sqlx::Pool<sqlx::Postgres>;
@@ -76,6 +79,7 @@ pub async fn update_flow_status_after_job_completion<
rsmq: Option<R>,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> error::Result<()> {
// this is manual tailrecursion because async_recursion blows up the stack
// todo!();
@@ -97,6 +101,8 @@ pub async fn update_flow_status_after_job_completion<
rsmq.clone(),
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
bench,
)
.await?;
while let Some(nrec) = rec {
@@ -117,6 +123,8 @@ pub async fn update_flow_status_after_job_completion<
rsmq.clone(),
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
bench,
)
.await
{
@@ -141,6 +149,8 @@ pub async fn update_flow_status_after_job_completion<
rsmq.clone(),
worker_name,
job_completed_tx.clone(),
#[cfg(feature = "benchmark")]
bench,
)
.await?
}
@@ -158,10 +168,7 @@ pub struct RecUpdateFlowStatusAfterJobCompletion {
}
#[derive(FromRow)]
pub struct SkipIfStopped {
pub skip_if_stopped: Option<bool>,
pub stop_early_expr: Option<String>,
pub continue_on_error: Option<bool>,
pub struct RowArgs {
pub args: Option<Json<HashMap<String, Box<RawValue>>>>,
}
@@ -173,6 +180,7 @@ struct RecoveryObject {
#[derive(sqlx::FromRow, Deserialize)]
pub struct RowFlowStatus {
pub flow_status: sqlx::types::Json<Box<serde_json::value::RawValue>>,
pub current_module: Option<sqlx::types::Json<Box<serde_json::value::RawValue>>>,
}
// #[instrument(level = "trace", skip_all)]
pub async fn update_flow_status_after_job_completion_internal<
@@ -193,7 +201,9 @@ pub async fn update_flow_status_after_job_completion_internal<
rsmq: Option<R>,
worker_name: &str,
job_completed_tx: Sender<SendResult>,
#[cfg(feature = "benchmark")] bench: &mut BenchmarkIter,
) -> error::Result<Option<RecUpdateFlowStatusAfterJobCompletion>> {
add_time!(bench, "update flow status internal START");
let (
should_continue_flow,
flow_job,
@@ -206,7 +216,7 @@ pub async fn update_flow_status_after_job_completion_internal<
// tracing::debug!("UPDATE FLOW STATUS: {flow:?} {success} {result:?} {w_id} {depth}");
let old_status_json = sqlx::query_as::<_, RowFlowStatus>(
"SELECT flow_status FROM queue WHERE id = $1 AND workspace_id = $2",
"SELECT flow_status, raw_flow->'modules'->(flow_status->'step')::int as current_module FROM queue WHERE id = $1 AND workspace_id = $2",
)
.bind(flow)
.bind(w_id)
@@ -225,6 +235,16 @@ pub async fn update_flow_status_after_job_completion_internal<
)))
})?;
let current_module = if let Some(x) = old_status_json.current_module {
Some(serde_json::from_str::<FlowModule>(x.0.get()).or_else(|e| {
Err(Error::InternalErr(format!(
"requiring current module to be parsable as FlowModule: {e:?}"
)))
})?)
} else {
None
};
let module_step = Step::from_i32_and_len(old_status.step, old_status.modules.len());
let module_status = match module_step {
@@ -247,9 +267,17 @@ pub async fn update_flow_status_after_job_completion_internal<
module_status,
FlowStatusModule::InProgress { iterator: Some(_), .. }
) {
let (loop_failures, parallelism) =
compute_skip_loop_failures_and_parallelism(flow, old_status.step, db).await?;
(true, loop_failures.unwrap_or(false), parallelism)
let value = current_module
.as_ref()
.and_then(|x| x.get_value_with_skip_failures().ok());
(
true,
value
.as_ref()
.and_then(|x| x.skip_failures)
.unwrap_or(false),
value.as_ref().and_then(|x| x.parallelism),
)
} else {
(false, false, None)
};
@@ -267,37 +295,43 @@ pub async fn update_flow_status_after_job_completion_internal<
stop_early_override
{
//do not stop early if module is a flow step
let flow_job = get_queued_job(&flow, w_id, db)
.await?
.ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
let module = get_module(&flow_job, &module_step);
let step = match module_step {
Step::PreprocessorStep => None,
Step::FailureStep => None,
Step::Step(i) => Some(i),
};
if module.is_some_and(|x| x.is_flow()) {
let is_flow = if let Some(step) = step {
sqlx::query_scalar!(
"SELECT raw_flow->'modules'->($1)->'value'->>'type' = 'flow' FROM queue WHERE id = $2",
step as i32,
&flow
)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!("error during retrieval of step's type: {e:#}"))
})?
.unwrap_or(false)
} else {
false
};
if is_flow {
(false, false, false)
} else {
(true, se, false)
}
} else if is_failure_step || matches!(module_step, Step::PreprocessorStep) {
(false, false, false)
} else {
let r = sqlx::query_as::<_, SkipIfStopped>(
"SELECT
raw_flow->'modules'->$1::int->'stop_after_if'->>'expr' as stop_early_expr,
(raw_flow->'modules'->$1::int->'stop_after_if'->>'skip_if_stopped')::bool as skip_if_stopped,
(raw_flow->'modules'->$1::int->'continue_on_error')::bool as continue_on_error,
args
FROM queue
WHERE id = $2"
)
.bind(old_status.step)
.bind(flow)
.fetch_one(db)
.await
.map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e:#}")))?;
} else if let Some(current_module) = current_module.as_ref() {
let stop_early = success
&& !is_branch_all
&& if let Some(expr) = r.stop_early_expr.clone() {
&& if let Some(ref expr) = current_module
.stop_after_if
.as_ref()
.map(|x| x.expr.clone())
{
let all_iters = match &module_status {
FlowStatusModule::InProgress { flow_jobs: Some(flow_jobs), .. }
if expr.contains("all_iters") =>
@@ -308,14 +342,22 @@ pub async fn update_flow_status_after_job_completion_internal<
}
_ => None,
};
let args = sqlx::query_as::<_, RowArgs>(
"SELECT
args
FROM queue
WHERE id = $2",
)
.bind(old_status.step)
.bind(flow)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!("retrieval of args from state: {e:#}"))
})?;
compute_bool_from_expr(
expr,
Marc::new(
r.args
.map(|x| x.0)
.unwrap_or_else(|| serde_json::from_str("{}").unwrap())
.to_owned(),
),
expr.to_string(),
Marc::new(args.args.unwrap_or_default().0),
result.clone(),
all_iters,
None,
@@ -329,9 +371,15 @@ pub async fn update_flow_status_after_job_completion_internal<
};
(
stop_early,
r.skip_if_stopped.unwrap_or(false),
r.continue_on_error.unwrap_or(false),
current_module
.stop_after_if
.as_ref()
.map(|x| x.skip_if_stopped)
.unwrap_or(false),
current_module.continue_on_error.unwrap_or(false),
)
} else {
(false, false, false)
};
let skip_branch_failure = match module_status {
@@ -340,26 +388,23 @@ pub async fn update_flow_status_after_job_completion_internal<
parallel,
..
} => compute_skip_branchall_failure(
flow,
job_id_for_status,
old_status.step,
*branch,
*parallel,
db,
current_module.as_ref(),
)
.await?
.unwrap_or(false),
_ => false,
};
let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
if matches!(module_step, Step::PreprocessorStep) {
sqlx::query!(
"UPDATE queue SET args = (select result FROM completed_job WHERE id = $1) WHERE id = $2",
job_id_for_status,
flow
).execute(&mut tx).await.map_err(|e| {
).execute(db).await.map_err(|e| {
Error::InternalErr(format!("error while updating args in preprocessing step: {e:#}"))
})?;
@@ -367,7 +412,7 @@ pub async fn update_flow_status_after_job_completion_internal<
r#"UPDATE completed_job SET args = '{"reason":"PREPROCESSOR_ARGS_ARE_DISCARDED"}'::jsonb WHERE id = $1"#,
job_id_for_status
)
.execute(&mut tx)
.execute(db)
.await
.map_err(|e| {
Error::InternalErr(format!(
@@ -376,6 +421,10 @@ pub async fn update_flow_status_after_job_completion_internal<
})?;
}
let mut tx: QueueTransaction<'_, _> = (rsmq.clone(), db.begin().await?).into();
add_time!(bench, "process module status START");
let (inc_step_counter, new_status) = match module_status {
FlowStatusModule::InProgress {
iterator,
@@ -387,25 +436,38 @@ pub async fn update_flow_status_after_job_completion_internal<
} if *parallel => {
let (nindex, len) = match (iterator, branchall) {
(Some(Iterator { itered, .. }), _) => {
set_success_in_flow_job_success(
flow_jobs_success,
jobs,
job_id_for_status,
&old_status,
flow,
success,
&mut tx,
)
.await?;
let position = if flow_jobs_success.is_some() {
find_flow_job_index(jobs, job_id_for_status)
} else {
None
};
let nindex = sqlx::query_scalar!(
let nindex = if let Some(position) = position {
sqlx::query_scalar!(
"UPDATE queue
SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb)
SET flow_status = JSONB_SET(
JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),
ARRAY['modules', $1::TEXT, 'iterator', 'index'],
((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb
),
last_ping = NULL
WHERE id = $2
RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
old_status.step,
flow
)
flow,
position as i32,
json!(success)
)} else {
sqlx::query_scalar!(
"UPDATE queue
SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'iterator', 'index'], ((flow_status->'modules'->$1::int->'iterator'->>'index')::int + 1)::text::jsonb),
last_ping = NULL
WHERE id = $2
RETURNING (flow_status->'modules'->$1::int->'iterator'->>'index')::int",
old_status.step,
flow
)
}
.fetch_one(&mut tx)
.await.map_err(|e| {
Error::InternalErr(format!(
@@ -421,24 +483,35 @@ pub async fn update_flow_status_after_job_completion_internal<
(nindex, itered.len() as i32)
}
(_, Some(BranchAllStatus { len, .. })) => {
set_success_in_flow_job_success(
flow_jobs_success,
jobs,
job_id_for_status,
&old_status,
flow,
success,
&mut tx,
)
.await?;
let nindex = sqlx::query_scalar!(
let position = if flow_jobs_success.is_some() {
find_flow_job_index(jobs, job_id_for_status)
} else {
None
};
let nindex = if let Some(position) = position {
sqlx::query_scalar!(
"UPDATE queue
SET flow_status = JSONB_SET(
JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4),
ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),
last_ping = NULL
WHERE id = $2
RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
old_status.step,
flow,
position as i32,
json!(success)
)
} else { sqlx::query_scalar!(
"UPDATE queue
SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb)
SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'branchall', 'branch'], ((flow_status->'modules'->$1::int->'branchall'->>'branch')::int + 1)::text::jsonb),
last_ping = NULL
WHERE id = $2
RETURNING (flow_status->'modules'->$1::int->'branchall'->>'branch')::int",
old_status.step,
flow
)
)}
.fetch_one(&mut tx)
.await
.map_err(|e| {
@@ -517,9 +590,9 @@ pub async fn update_flow_status_after_job_completion_internal<
tracing::info!(
"parallel iteration {job_id_for_status} of flow {flow} has finished",
);
(true, Some(new_status))
} else {
add_time!(bench, "handle parallel flow start");
tx.commit().await?;
if parallelism.is_some() {
@@ -537,18 +610,6 @@ pub async fn update_flow_status_after_job_completion_internal<
})?;
}
sqlx::query!(
"UPDATE queue
SET last_ping = null
WHERE id = $1",
flow
)
.execute(db)
.await
.map_err(|e| {
Error::InternalErr(format!("error while setting last ping to null: {e:#}"))
})?;
let r = sqlx::query_scalar!(
"DELETE FROM parallel_monitor_lock WHERE parent_flow_id = $1 and job_id = $2 RETURNING last_ping",
flow,
@@ -562,7 +623,7 @@ pub async fn update_flow_status_after_job_completion_internal<
r.unwrap()
);
}
add_time!(bench, "non final parallel flow finished");
return Ok(None);
}
}
@@ -652,25 +713,12 @@ pub async fn update_flow_status_after_job_completion_internal<
)
} else {
let inc = if continue_on_error {
let retry = sqlx::query_scalar!(
"SELECT raw_flow->'modules'->$2::int->'retry' FROM queue WHERE id = $1",
flow,
old_status.step
)
.fetch_optional(&mut tx)
.await
.map_err(|e| {
Error::InternalErr(format!(
"error while getting retry from step: {e:#}"
))
})?
.flatten();
let retry = retry
.map(|x| serde_json::from_value::<Retry>(x).ok())
.flatten()
let retry = current_module
.as_ref()
.and_then(|x| x.retry.clone())
.unwrap_or_default();
tracing::info!("update flow status on rety: {retry:#?} ");
tracing::info!("update flow status on rety: {retry:#?} ");
next_retry(&retry, &old_status.retry).is_none()
} else {
false
@@ -802,30 +850,27 @@ pub async fn update_flow_status_after_job_completion_internal<
match &new_status {
Some(FlowStatusModule::Success { .. }) if is_loop || is_branch_all => {
let r_after_all_iters = sqlx::query_as::<_, SkipIfStopped>(
"SELECT
raw_flow->'modules'->$1::int->'stop_after_all_iters_if'->>'expr' as stop_early_expr,
(raw_flow->'modules'->$1::int->'stop_after_all_iters_if'->>'skip_if_stopped')::bool as skip_if_stopped,
NULL as continue_on_error,
if let Some(ref expr) = current_module
.as_ref()
.and_then(|m| m.stop_after_all_iters_if.as_ref().map(|x| x.expr.clone()))
{
let args = sqlx::query_as::<_, RowArgs>(
"SELECT
args
FROM queue
WHERE id = $2"
WHERE id = $2",
)
.bind(old_status.step)
.bind(flow)
.fetch_one(db)
.await
.map_err(|e| Error::InternalErr(format!("retrieval of stop_early_expr from state: {e:#}")))?;
if let Some(expr) = r_after_all_iters.stop_early_expr {
.map_err(|e| {
Error::InternalErr(format!("retrieval of args from state: {e:#}"))
})?;
let should_stop = compute_bool_from_expr(
expr,
Marc::new(
r_after_all_iters
.args
.map(|x| x.0)
.unwrap_or_else(|| serde_json::from_str("{}").unwrap())
.to_owned(),
),
expr.to_string(),
Marc::new(args.args.unwrap_or_default().0),
nresult.clone(),
None,
None,
@@ -837,7 +882,14 @@ pub async fn update_flow_status_after_job_completion_internal<
if should_stop {
stop_early = should_stop;
skip_if_stop_early = r_after_all_iters.skip_if_stopped.unwrap_or(false);
skip_if_stop_early = current_module
.as_ref()
.and_then(|m| {
m.stop_after_all_iters_if
.as_ref()
.map(|x| x.skip_if_stopped)
})
.unwrap_or(false);
}
}
}
@@ -862,6 +914,7 @@ pub async fn update_flow_status_after_job_completion_internal<
let flow_job = get_queued_job_tx(flow, w_id, tx.transaction_mut())
.await?
.ok_or_else(|| Error::InternalErr(format!("requiring flow to be in the queue")))?;
tx.commit().await?;
let job_root = flow_job
.root_job
@@ -894,14 +947,13 @@ pub async fn update_flow_status_after_job_completion_internal<
false
if !is_failure_step
&& !skip_error_handler
&& has_failure_module(flow, tx.transaction_mut()).await? =>
&& has_failure_module(flow, db).await? =>
{
true
}
false => false,
};
tx.commit().await?;
tracing::debug!(id = %flow_job.id, root_id = %job_root, "flow status updated");
(
@@ -964,6 +1016,8 @@ pub async fn update_flow_status_after_job_completion_internal<
rsmq.clone(),
worker_name,
true,
#[cfg(feature = "benchmark")]
bench,
)
.await?;
} else {
@@ -994,6 +1048,8 @@ pub async fn update_flow_status_after_job_completion_internal<
let success = success
&& (!is_failure_step || result_has_recover_true(nresult.clone()))
&& !skip_error_handler;
add_time!(bench, "flow status update 1");
if success {
add_completed_job(
db,
@@ -1005,6 +1061,8 @@ pub async fn update_flow_status_after_job_completion_internal<
None,
rsmq.clone(),
true,
#[cfg(feature = "benchmark")]
bench,
)
.await?;
} else {
@@ -1022,6 +1080,8 @@ pub async fn update_flow_status_after_job_completion_internal<
None,
rsmq.clone(),
true,
#[cfg(feature = "benchmark")]
bench,
)
.await?;
}
@@ -1059,6 +1119,8 @@ pub async fn update_flow_status_after_job_completion_internal<
rsmq.clone(),
worker_name,
true,
#[cfg(feature = "benchmark")]
bench,
)
.await;
true
@@ -1096,6 +1158,10 @@ pub async fn update_flow_status_after_job_completion_internal<
}
}
fn find_flow_job_index(flow_jobs: &Vec<Uuid>, job_id_for_status: &Uuid) -> Option<usize> {
flow_jobs.iter().position(|x| x == job_id_for_status)
}
async fn set_success_in_flow_job_success<'c, R: rsmq_async::RsmqConnection + Send>(
flow_jobs_success: &Option<Vec<Option<bool>>>,
flow_jobs: &Vec<Uuid>,
@@ -1105,10 +1171,8 @@ async fn set_success_in_flow_job_success<'c, R: rsmq_async::RsmqConnection + Sen
success: bool,
tx: &mut QueueTransaction<'c, R>,
) -> error::Result<()> {
let flow_jobs_success = flow_jobs_success.clone();
if flow_jobs_success.is_some() {
let position = flow_jobs.iter().position(|x| x == job_id_for_status);
let position = find_flow_job_index(flow_jobs, job_id_for_status);
if let Some(position) = position {
sqlx::query!(
"UPDATE queue SET flow_status = JSONB_SET(flow_status, ARRAY['modules', $1::TEXT, 'flow_jobs_success', $3::TEXT], $4) WHERE id = $2",
@@ -1172,31 +1236,12 @@ fn get_module(flow_job: &QueuedJob, module_step: &Step) -> Option<FlowModule> {
}
}
async fn compute_skip_loop_failures_and_parallelism(
flow: Uuid,
step: i32,
db: &DB,
) -> Result<(Option<bool>, Option<i32>), Error> {
sqlx::query_as(
"SELECT (raw_flow->'modules'->$1->'value'->>'skip_failures')::bool, (raw_flow->'modules'->$1->'value'->>'parallelism')::int
FROM queue
WHERE id = $2",
)
.bind(step)
.bind(flow)
.fetch_one(db)
.await
.map(|(v, n)| (v,n))
.map_err(|e| Error::InternalErr(format!("error during retrieval of skip_loop_failures: {e:#}")))
}
async fn compute_skip_branchall_failure<'c>(
flow: Uuid,
job: &Uuid,
step: i32,
branch: usize,
parallel: bool,
db: &DB,
flow_module: Option<&FlowModule>,
) -> Result<Option<bool>, Error> {
let branch = if parallel {
sqlx::query_scalar!("SELECT script_path FROM completed_job WHERE id = $1", job)
@@ -1220,35 +1265,23 @@ async fn compute_skip_branchall_failure<'c>(
} else {
branch as i32
};
sqlx::query_as(
"SELECT (raw_flow->'modules'->$1->'value'->'branches'->$2->>'skip_failure')::bool
FROM queue
WHERE id = $3",
)
.bind(step)
.bind(branch)
.bind(flow)
.fetch_one(db)
.await
.map(|(v,)| v)
.map_err(|e| {
Error::InternalErr(format!(
"error during retrieval of skip_loop_failures: {e:#}"
))
})
Ok(flow_module
.and_then(|x| x.get_branches_skip_failures().ok())
.and_then(|x| {
x.branches
.get(branch as usize)
.map(|x| x.skip_failure.unwrap_or(false))
}))
}
async fn has_failure_module<'c>(
flow: Uuid,
tx: &mut sqlx::Transaction<'c, sqlx::Postgres>,
) -> Result<bool, Error> {
async fn has_failure_module<'c>(flow: Uuid, db: &DB) -> Result<bool, Error> {
sqlx::query_scalar::<_, Option<bool>>(
"SELECT raw_flow->'failure_module' != 'null'::jsonb
FROM queue
WHERE id = $1",
)
.bind(flow)
.fetch_one(&mut **tx)
.fetch_one(db)
.await
.map_err(|e| {
Error::InternalErr(format!(
backend/windmill-worker/src/worker_lockfiles.rs
+31
View File
@@ -25,6 +25,7 @@ use windmill_parser_py_imports::parse_relative_imports;
use windmill_parser_ts::parse_expr_for_imports;
use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel};
use crate::common::OccupancyMetrics;
use crate::python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile};
use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile};
use crate::{
@@ -212,6 +213,7 @@ pub async fn handle_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync +
base_internal_url: &str,
token: &str,
rsmq: Option<R>,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<Box<RawValue>> {
let raw_code = match job.raw_code {
Some(ref code) => code.to_owned(),
@@ -273,6 +275,7 @@ pub async fn handle_dependency_job<R: rsmq_async::RsmqConnection + Send + Sync +
script_path,
raw_deps,
npm_mode,
occupancy_metrics,
)
.await;
@@ -533,6 +536,7 @@ pub async fn handle_flow_dependency_job<R: rsmq_async::RsmqConnection + Send + S
base_internal_url: &str,
token: &str,
rsmq: Option<R>,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<Box<serde_json::value::RawValue>> {
let job_path = job.script_path.clone().ok_or_else(|| {
error::Error::InternalErr(
@@ -605,6 +609,7 @@ pub async fn handle_flow_dependency_job<R: rsmq_async::RsmqConnection + Send + S
base_internal_url,
token,
&nodes_to_relock,
occupancy_metrics,
)
.await?;
let new_flow_value = serde_json::to_value(flow).map_err(to_anyhow)?;
@@ -709,6 +714,7 @@ async fn lock_modules<'c>(
base_internal_url: &str,
token: &str,
locks_to_reload: &Option<Vec<String>>,
occupancy_metrics: &mut OccupancyMetrics,
// (modules to replace old seq (even unmmodified ones), new transaction, modified ids) )
) -> Result<(
Vec<FlowModule>,
@@ -754,6 +760,7 @@ async fn lock_modules<'c>(
base_internal_url,
token,
locks_to_reload,
occupancy_metrics,
))
.await?;
e.value = FlowModuleValue::ForloopFlow {
@@ -785,6 +792,7 @@ async fn lock_modules<'c>(
base_internal_url,
token,
locks_to_reload,
occupancy_metrics,
))
.await?;
nmodified_ids.extend(inner_modified_ids);
@@ -809,6 +817,7 @@ async fn lock_modules<'c>(
base_internal_url,
token,
locks_to_reload,
occupancy_metrics,
))
.await?;
e.value =
@@ -835,6 +844,7 @@ async fn lock_modules<'c>(
base_internal_url,
token,
locks_to_reload,
occupancy_metrics,
))
.await?;
nmodified_ids.extend(inner_modified_ids);
@@ -856,6 +866,7 @@ async fn lock_modules<'c>(
base_internal_url,
token,
locks_to_reload,
occupancy_metrics,
))
.await?;
e.value = FlowModuleValue::BranchOne { branches: nbranches, default: ndefault }
@@ -904,6 +915,7 @@ async fn lock_modules<'c>(
),
false,
None,
occupancy_metrics,
)
.await;
//
@@ -1014,6 +1026,7 @@ async fn lock_modules_app(
job_path: &str,
base_internal_url: &str,
token: &str,
occupancy_metrics: &mut OccupancyMetrics,
) -> Result<Value> {
match value {
Value::Object(mut m) => {
@@ -1058,6 +1071,7 @@ async fn lock_modules_app(
&format!("{}/app", job.script_path()),
false,
None,
occupancy_metrics,
)
.await;
match new_lock {
@@ -1113,6 +1127,7 @@ async fn lock_modules_app(
job_path,
base_internal_url,
token,
occupancy_metrics,
)
.await?,
);
@@ -1135,6 +1150,7 @@ async fn lock_modules_app(
job_path,
base_internal_url,
token,
occupancy_metrics,
)
.await?,
);
@@ -1156,6 +1172,7 @@ pub async fn handle_app_dependency_job<R: rsmq_async::RsmqConnection + Send + Sy
base_internal_url: &str,
token: &str,
rsmq: Option<R>,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<()> {
let job_path = job.script_path.clone().ok_or_else(|| {
error::Error::InternalErr(
@@ -1185,6 +1202,7 @@ pub async fn handle_app_dependency_job<R: rsmq_async::RsmqConnection + Send + Sy
&job_path,
base_internal_url,
token,
occupancy_metrics,
)
.await?;
@@ -1259,6 +1277,7 @@ async fn python_dep(
worker_name: &str,
w_id: &str,
worker_dir: &str,
occupancy_metrics: &mut Option<&mut OccupancyMetrics>,
) -> std::result::Result<String, Error> {
create_dependencies_dir(job_dir).await;
let req: std::result::Result<String, Error> = pip_compile(
@@ -1270,6 +1289,7 @@ async fn python_dep(
db,
worker_name,
w_id,
occupancy_metrics,
)
.await;
// install the dependencies to pre-fill the cache
@@ -1284,6 +1304,7 @@ async fn python_dep(
worker_name,
job_dir,
worker_dir,
occupancy_metrics,
)
.await;
@@ -1313,6 +1334,7 @@ async fn capture_dependency_job(
script_path: &str,
raw_deps: bool,
npm_mode: Option<bool>,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<String> {
match job_language {
ScriptLang::Python3 => {
@@ -1342,6 +1364,7 @@ async fn capture_dependency_job(
worker_name,
w_id,
worker_dir,
&mut Some(occupancy_metrics),
)
.await
}
@@ -1364,6 +1387,7 @@ async fn capture_dependency_job(
worker_name,
w_id,
worker_dir,
&mut Some(occupancy_metrics),
)
.await
}
@@ -1385,6 +1409,7 @@ async fn capture_dependency_job(
false,
worker_name,
w_id,
occupancy_metrics,
)
.await
}
@@ -1404,6 +1429,7 @@ async fn capture_dependency_job(
w_id,
worker_name,
base_internal_url,
&mut Some(occupancy_metrics),
)
.await
}
@@ -1431,6 +1457,7 @@ async fn capture_dependency_job(
None
},
npm_mode,
&mut Some(occupancy_metrics),
)
.await?;
if req.is_some() && !raw_deps {
@@ -1445,6 +1472,7 @@ async fn capture_dependency_job(
base_internal_url,
worker_name,
&token,
occupancy_metrics,
)
.await?;
}
@@ -1475,6 +1503,7 @@ async fn capture_dependency_job(
worker_name,
reqs,
None,
occupancy_metrics,
)
.await
}
@@ -1494,6 +1523,7 @@ async fn capture_dependency_job(
db,
worker_name,
w_id,
occupancy_metrics,
)
.await?;
@@ -1507,6 +1537,7 @@ async fn capture_dependency_job(
w_id,
base_internal_url,
&compute_rust_hash(&job_raw_code, Some(&lockfile)),
occupancy_metrics,
)
.await?;
Ok(lockfile)
benchmarks/lib.ts
+1 -1
View File
@@ -2,7 +2,7 @@ import { sleep } from "https://deno.land/x/sleep@v1.2.1/mod.ts";
import * as windmill from "https://deno.land/x/windmill@v1.174.0/mod.ts";
import * as api from "https://deno.land/x/windmill@v1.174.0/windmill-api/index.ts";
export const VERSION = "v1.401.0";
export const VERSION = "v1.402.3";
export async function login(email: string, password: string): Promise<string> {
return await windmill.UserService.login({
cli/gen/core/OpenAPI.ts
+1 -1
View File
@@ -54,7 +54,7 @@ export const OpenAPI: OpenAPIConfig = {
PASSWORD: undefined,
TOKEN: getEnv("WM_TOKEN"),
USERNAME: undefined,
VERSION: '1.398.1',
VERSION: '1.401.0',
WITH_CREDENTIALS: true,
interceptors: {
request: new Interceptors(),
cli/gen/services.gen.ts
+18 -1
View File
File diff suppressed because one or more lines are too long
cli/gen/types.gen.ts
+20
View File
@@ -160,6 +160,7 @@ export type QueuedJob = {
priority?: number;
self_wait_time_ms?: number;
aggregate_wait_time_ms?: number;
suspend?: number;
};
export type job_kind = 'script' | 'preview' | 'dependencies' | 'flowdependencies' | 'appdependencies' | 'flow' | 'flowpreview' | 'script_hub' | 'identity' | 'deploymentcallback' | 'singlescriptflow';
@@ -613,6 +614,9 @@ export type WorkerPing = {
last_job_id?: string;
last_job_workspace_id?: string;
occupancy_rate?: number;
occupancy_rate_15s?: number;
occupancy_rate_5m?: number;
occupancy_rate_30m?: number;
memory?: number;
vcpus?: number;
memory_usage?: number;
@@ -4406,6 +4410,22 @@ export type ListJobsResponse = (Array<Job>);
export type GetDbClockResponse = (number);
export type CountJobsByTagData = {
/**
* Past Time horizon in seconds (when to start the count = now - horizon) (default is 3600)
*/
horizonSecs?: number;
/**
* Specific workspace ID to filter results (optional)
*/
workspaceId?: string;
};
export type CountJobsByTagResponse = (Array<{
tag: string;
count: number;
}>);
export type GetJobData = {
id: string;
noLogs?: boolean;
cli/instance.ts
+199 -77
View File
@@ -6,6 +6,7 @@ import {
yamlParse,
Command,
setClient,
Table,
} from "./deps.ts";
import * as wmill from "./gen/services.gen.ts";
@@ -62,39 +63,49 @@ export async function allInstances(): Promise<Instance[]> {
return [];
}
}
export async function addInstance() {
let remote = await Input.prompt({
message: "Enter the remote url of this instance",
default: "https://app.windmill.dev/",
});
remote = new URL(remote).toString(); // add trailing slash in all cases!
export async function addInstance(
opts: {},
instanceName: string | undefined,
remote: string | undefined,
token: string | undefined
) {
if (!remote) {
remote = await Input.prompt({
message: "Enter the remote url of this instance",
default: "https://my.windmill.dev/",
});
remote = new URL(remote).toString(); // add trailing slash in all cases!
}
const defaultName = new URL(remote).hostname;
if (!instanceName) {
const defaultName = new URL(remote).hostname.split(".")[0];
const name = await Input.prompt({
message: "Enter a name for this instance",
default: defaultName,
});
instanceName = await Input.prompt({
message: "Enter a name for this instance",
default: defaultName,
});
}
const prefix = instanceName.toLowerCase().replace(/[^a-z0-9]/g, "");
const prefix = name.toLowerCase().replace(/[^a-z0-9]/g, "");
let token: string | undefined = undefined;
while (!token) {
token = await loginInteractive(remote);
}
await appendInstance({
name,
name: instanceName,
remote,
token,
prefix,
});
log.info(
colors.green.underline(`Added instance ${name} with remote ${remote}!`)
colors.green.underline(
`Added instance ${instanceName} with remote ${remote}!`
)
);
await switchI({}, instanceName);
return {
name,
name: instanceName,
remote,
token,
prefix,
@@ -160,37 +171,59 @@ export function compareInstanceObjects<T extends string>(
return changes;
}
type InstanceSyncOptions = {
export type InstanceSyncOptions = {
skipUsers?: boolean;
skipSettings?: boolean;
skipConfigs?: boolean;
skipGroups?: boolean;
includeWorkspaces?: boolean;
instance?: string;
baseUrl?: string;
token?: string;
yes?: boolean;
};
async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
export async function pickInstance(opts: InstanceSyncOptions, allowNew: boolean) {
const instances = await allInstances();
let instance: Instance;
if (instances.length < 1) {
instance = await addInstance();
} else {
const choice = (await Select.prompt({
message: "Select an instance to pull from",
options: [
...instances.map((i) => ({
name: `${i.name} (${i.remote})`,
value: i.name,
})),
{ name: "Add new instance", value: "new" },
],
})) as unknown as string;
if (choice === "new") {
instance = await addInstance();
if (opts.baseUrl && opts.token) {
log.info("Using instance fully defined by --base-url and --token")
return {
name: "custom",
remote: opts.baseUrl,
token: opts.token,
prefix: "custom",
};
}
if (!allowNew && instances.length < 1) {
throw new Error("No instance found, please add one first");
}
const instanceName = await getActiveInstance(opts);
let instance: Instance | undefined = instances.find(
(i) => i.name === instanceName
);
if (!instance) {
if (instances.length < 1) {
instance = await addInstance({}, undefined, undefined, undefined);
} else {
instance = instances.find((i) => i.name === choice)!;
const choice = (await Select.prompt({
message: "Select an instance",
options: [
...instances.map((i) => ({
name: `${i.name} (${i.remote})`,
value: i.name,
})),
{ name: "Add new instance", value: "new" },
],
})) as unknown as string;
if (choice === "new") {
instance = await addInstance({}, undefined, undefined, undefined);
} else {
instance = instances.find((i) => i.name === choice)!;
}
}
} else {
log.info(`Selected instance: ${instance.name}`);
}
setClient(
@@ -198,6 +231,10 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
instance.remote.slice(0, instance.remote.length - 1)
);
return instance;
}
async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
const instance = await pickInstance(opts, true);
log.info("Pulling instance-level changes");
log.info(`remote (${instance.name}) -> local`);
@@ -221,10 +258,13 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
const totalChanges = uChanges + sChanges + cChanges + gChanges;
if (totalChanges > 0) {
const confirm = await Confirm.prompt({
message: `Do you want to apply these ${totalChanges} instance-level changes?`,
default: true,
});
let confirm = true;
if (opts.yes !== true) {
confirm = await Confirm.prompt({
message: `Do you want to pull these ${totalChanges} instance-level changes?`,
default: true,
});
}
if (confirm) {
if (!opts.skipUsers && uChanges > 0) {
@@ -318,32 +358,7 @@ async function instancePull(opts: GlobalOptions & InstanceSyncOptions) {
async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
let instances = await allInstances();
let instance: Instance;
if (instances.length < 1) {
instance = await addInstance();
} else {
const choice = (await Select.prompt({
message: "Select an instance to push to",
options: [
...instances.map((i) => ({
name: `${i.name} (${i.remote})`,
value: i.name,
})),
{ name: "Add new instance", value: "new" },
],
})) as unknown as string;
if (choice === "new") {
instance = await addInstance();
} else {
instance = instances.find((i) => i.name === choice)!;
}
}
setClient(
instance.token,
instance.remote.slice(0, instance.remote.length - 1)
);
const instance = await pickInstance(opts, true);
log.info("Pushing instance-level changes");
log.info!(`remote (${instance.name}) <- local`);
@@ -368,10 +383,13 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
const totalChanges = uChanges + sChanges + cChanges + gChanges;
if (totalChanges > 0) {
const confirm = await Confirm.prompt({
message: `Do you want to apply these ${totalChanges} instance-level changes?`,
default: true,
});
let confirm = true;
if (opts.yes !== true) {
confirm = await Confirm.prompt({
message: `Do you want to apply these ${totalChanges} instance-level changes?`,
default: true,
});
}
if (confirm) {
if (!opts.skipUsers && uChanges > 0) {
@@ -485,36 +503,140 @@ async function instancePush(opts: GlobalOptions & InstanceSyncOptions) {
}
}
async function switchI(opts: {}, instanceName: string) {
const all = await allInstances();
if (all.findIndex((x) => x.name === instanceName) === -1) {
log.info(
colors.red.bold(`! This instance ${instanceName} does not exist locally.`)
);
log.info("available instances:");
for (const w of all) {
log.info(" - " + w.name);
}
return;
}
await Deno.writeTextFile(
(await getRootStore()) + "/activeInstance",
instanceName
);
log.info(colors.green.underline(`Switched to instance ${instanceName}`));
}
export async function getActiveInstance(opts: {
instance?: string;
}): Promise<string | undefined> {
if (opts.instance) {
return opts.instance;
}
try {
return await Deno.readTextFile((await getRootStore()) + "/activeInstance");
} catch {
return undefined;
}
}
async function whoami(opts: {}) {
await pickInstance({}, false);
try {
const whoamiInfo = await wmill.globalWhoami();
log.info(colors.green.underline(`global whoami infos:`));
log.info(JSON.stringify(whoamiInfo, null, 2));
} catch (error) {
log.error(colors.red(`Failed to retrieve whoami information: ${error.message}`));
}
}
const command = new Command()
.description(
"sync local with a remote instance or the opposite (push or pull)"
)
.action(() =>
log.info("2 actions available, pull and push. Use -h to display help.")
)
.action(async () => {
log.info(
"4 actions available, add, remove, switch, pull and push. Use -h to display help."
);
const activeInstance = await getActiveInstance({});
new Table()
.header(["name", "remote", "token"])
.padding(2)
.border(true)
.body(
(await allInstances()).map((x) => [
x.name === activeInstance ? colors.underline(x.name) : x.name,
x.remote,
x.token.substring(0, 7) + "***",
])
)
.render();
if (activeInstance) {
log.info(`Selected instance: ${activeInstance}`);
} else {
log.info("No active instance selected");
}
log.info("Use 'wmill instance add' to add a new instance");
})
.command("add")
.description("Add a new instance")
.action(addInstance as any)
.arguments("[instance_name:string] [remote:string] [token:string]")
.command("remove")
.description("Remove an instance")
.complete("instance", async () => (await allInstances()).map((x) => x.name))
.arguments("<instance:string:instance>")
.action(async (instance) => {
const instances = await allInstances();
const choice = (await Select.prompt({
message: "Select an instance to remove",
options: instances.map((i) => ({
name: `${i.name} (${i.remote})`,
value: i.name,
})),
})) as unknown as string;
await removeInstance(choice);
log.info(colors.green.underline(`Removed instance ${choice}`));
})
.command("switch")
.complete("instance", async () => (await allInstances()).map((x) => x.name))
.arguments("<instance:string:instance>")
.description("Switch the current instance")
.action(switchI as any)
.command("pull")
.description(
"Pull instance settings, users, configs, instance groups and overwrite local"
)
.option("--yes", "Pull without needing confirmation")
.option("--skip-users", "Skip pulling users")
.option("--skip-settings", "Skip pulling settings")
.option("--skip-configs", "Skip pulling configs (worker groups and SMTP)")
.option("--skip-groups", "Skip pulling instance groups")
.option("--include-workspaces", "Also pull workspaces")
.action(instancePull as any)
.command("push")
.description(
"Push instance settings, users, configs, group and overwrite remote"
)
.option("--yes", "Push without needing confirmation")
.option("--skip-users", "Skip pushing users")
.option("--skip-settings", "Skip pushing settings")
.option("--skip-configs", "Skip pushing configs (worker groups and SMTP)")
.option("--skip-groups", "Skip pushing instance groups")
.option("--include-workspaces", "Also push workspaces")
.option(
"--base-url",
"Base url to be passed to the instance settings instead of the local one"
"--instance",
"Name of the instance to push to, override the active instance"
)
.action(instancePush as any);
.option(
"--base-url",
"If used with --token, will be used as the base url for the instance"
)
.action(instancePush as any)
.command("whoami")
.description("Display information about the currently logged-in user")
.action(whoami as any);
export default command;
cli/main.ts
+9 -3
View File
@@ -19,6 +19,8 @@ import folder from "./folder.ts";
import schedule from "./schedule.ts";
import sync from "./sync.ts";
import instance from "./instance.ts";
import workerGroups from "./worker_groups.ts";
import dev from "./dev.ts";
import { fetchVersion } from "./context.ts";
import { GlobalOptions } from "./types.ts";
@@ -28,6 +30,8 @@ import { NpmProvider } from "./upgrade.ts";
import { pull as hubPull } from "./hub.ts";
import { pull, push } from "./sync.ts";
import { add as workspaceAdd } from "./workspace.ts";
import workers from "./workers.ts";
import queues from "./queues.ts";
export {
flow,
@@ -56,14 +60,14 @@ export {
// }
// });
export const VERSION = "1.401.0";
export const VERSION = "1.402.3";
const command = new Command()
.name("wmill")
.action(() =>
log.info(`Welcome to Windmill CLI ${VERSION}. Use -h for help.`)
)
.description("A simple CLI tool for windmill.")
.description("Windmill CLI")
.globalOption(
"--workspace <workspace:string>",
@@ -117,7 +121,9 @@ const command = new Command()
.command("dev", dev)
.command("sync", sync)
.command("instance", instance)
.command("worker-groups", workerGroups)
.command("workers", workers)
.command("queues", queues)
.command("version", "Show version information")
.action(async (opts) => {
console.log("CLI build against " + VERSION);
cli/queues.ts
+147
View File
@@ -0,0 +1,147 @@
import { Command, Table } from "./deps.ts";
import { log } from "./deps.ts";
import * as wmill from "./gen/services.gen.ts";
import { pickInstance } from "./instance.ts";
type Data = {
count: number;
later: number;
suspended: number;
waiting: number;
running: number;
rps30s: string;
rps5min: string;
rps30min: string;
rps24h: string;
}
type GlobalOptions = {
instance?: string;
baseUrl?: string;
};
function createRow(tag: string, data: Record<string, Data>) {
if (data[tag]) {
return;
} else {
data[tag] = {
count: 0,
waiting: 0,
later: 0,
running: 0,
suspended: 0,
rps30s: "",
rps5min: "",
rps30min: "",
rps24h: "",
}
}
}
async function displayQueues(opts: GlobalOptions, workspace?: string) {
const activeInstance = await pickInstance(opts, true);
if (activeInstance) {
try {
const queuedJobs = await wmill.listQueue({workspace: workspace ?? 'admins', allWorkspaces: workspace === undefined, perPage: 100000});
const jobCounts30s = await wmill.countJobsByTag({
horizonSecs: 30,
workspaceId: workspace,
});
const nowFromDb = new Date(await wmill.getDbClock());
const jobCounts5min = await wmill.countJobsByTag({
horizonSecs: 300,
workspaceId: workspace,
});
const jobCounts30min = await wmill.countJobsByTag({
horizonSecs: 1800,
workspaceId: workspace,
});
const jobCounts24h = await wmill.countJobsByTag({
horizonSecs: 86400,
workspaceId: workspace,
});
const data: Record<string, Data> = {}
for (const job of queuedJobs) {
createRow(job.tag, data);
const scheduledFor = new Date(job.scheduled_for ?? "");
if (job.running) {
if (job.suspend) {
data[job.tag].suspended += 1;
} else {
data[job.tag].running += 1;
}
} else if (scheduledFor <= nowFromDb) {
data[job.tag].waiting += 1;
} else {
data[job.tag].later += 1;
}
}
for (const count of jobCounts30s) {
const tag = count.tag;
createRow(tag, data);
data[tag].rps30s = (count.count / 30).toFixed(3);
}
for (const count of jobCounts5min) {
const tag = count.tag;
createRow(tag, data);
data[tag].rps5min = (count.count / 300).toFixed(3);
}
for (const count of jobCounts30min) {
const tag = count.tag;
createRow(tag, data);
data[tag].rps30min = (count.count / 1800).toFixed(3);
}
for (const count of jobCounts24h) {
const tag = count.tag;
createRow(tag, data);
data[tag].rps24h = (count.count / 86400).toFixed(3);
}
const table = new Table();
table.header([
"",
"Running",
"Waiting",
"Later",
"Suspended",
"RPS (30s)",
"RPS (5min)",
"RPS (30min)",
"RPS (24h)",
]);
let body = []
for (const tag in data) {
body.push([tag, data[tag].running, data[tag].waiting, data[tag].later, data[tag].suspended, data[tag].rps30s, data[tag].rps5min, data[tag].rps30min, data[tag].rps24h]);
}
table.body(body).render();
} catch (error) {
log.error("Failed to fetch queue metrics:", error);
}
} else {
log.info("No active instance found");
log.info("Use 'wmill instance add' to add a new instance");
}
}
const command = new Command()
.description("List all queues with their metrics")
.arguments("[workspace:string] the optional workspace to filter by (default to all workspaces)")
.option(
"--instance [instance]",
"Name of the instance to push to, override the active instance"
)
.option(
"--base-url [baseUrl]",
"If used with --token, will be used as the base url for the instance"
)
.action(displayQueues as any);
export default command;
cli/settings.ts
+14 -3
View File
@@ -8,6 +8,7 @@ import { isSuperset } from "./types.ts";
import { deepEqual } from "./utils.ts";
import * as wmill from "./gen/services.gen.ts";
import { Config, GlobalSetting } from "./gen/types.gen.ts";
import { removeWorkerPrefix } from "./worker_groups.ts";
export interface SimplifiedSettings {
// slack_team_id?: string;
@@ -354,7 +355,12 @@ export async function pushInstanceSettings(
}
export async function pullInstanceConfigs(preview = false) {
const remoteConfigs = await wmill.listConfigs();
const remoteConfigs = (await wmill.listConfigs()).map((x) => {
return {
...x,
name: removeWorkerPrefix(x.name),
};
});
if (preview) {
let localConfigs: Config[] = [];
@@ -383,7 +389,12 @@ export async function pullInstanceConfigs(preview = false) {
}
export async function pushInstanceConfigs(preview: boolean = false) {
const remoteConfigs = await wmill.listConfigs();
const remoteConfigs = (await wmill.listConfigs()).map((x) => {
return {
...x,
name: removeWorkerPrefix(x.name),
};
});
const localConfigs = (await Deno.readTextFile("instance_configs.yaml")
.then((raw) => yamlParse(raw))
.catch(() => [])) as Config[];
@@ -404,7 +415,7 @@ export async function pushInstanceConfigs(preview: boolean = false) {
}
try {
await wmill.updateConfig({
name: config.name,
name: config.name.startsWith('worker__') ? config.name : `worker__${config.name}`,
requestBody: config.config,
});
} catch (err) {
cli/worker_groups.ts
+138
View File
@@ -0,0 +1,138 @@
import { Command, Confirm, setClient, Table } from "./deps.ts";
import { log } from "./deps.ts";
import { allInstances, getActiveInstance, Instance, InstanceSyncOptions, pickInstance } from "./instance.ts";
import * as wmill from "./gen/services.gen.ts";
import { pullInstanceConfigs, pushInstanceConfigs } from "./settings.ts";
type GlobalOptions = {
instance?: string;
baseUrl?: string;
};
export async function getInstance(opts: GlobalOptions) {
const instances = await allInstances();
const instanceName = await getActiveInstance(opts);
const instance = instances.find((i) => i.name === instanceName);
if (instance) {
setClient(
instance.token,
instance.remote.slice(0, instance.remote.length - 1)
);
}
return instance;
}
export function removeWorkerPrefix(name: string) {
if (name.startsWith("worker__")) {
return name.substring(8);
}
return name;
}
export async function displayWorkerGroups(opts: void) {
log.info("2 actions available, pull and push.");
const activeInstance = await getActiveInstance({});
if (activeInstance) {
log.info("Active instance: " + activeInstance);
const instance = await getInstance({});
if (instance) {
const wGroups = await wmill.listWorkerGroups();
new Table()
.header(["name", "config"])
.padding(2)
.border(true)
.body(wGroups.map((x) => [removeWorkerPrefix(x.name), JSON.stringify(x.config, null, 2)]))
.render();
} else {
log.error(`Instance ${activeInstance} not found`);
}
} else {
log.info("No active instance found");
log.info("Use 'wmill instance add' to add a new instance");
}
}
async function pullWorkerGroups(opts: InstanceSyncOptions) {
await pickInstance(opts, true);
const totalChanges = await pullInstanceConfigs(true) ?? 0;
if (totalChanges === 0) {
log.info("No changes to apply");
return;
}
let confirm = true;
if (opts.yes !== true) {
confirm = await Confirm.prompt({
message: `Do you want to pul these ${totalChanges} instance-level changes?`,
default: true,
});
}
if (confirm) {
await pullInstanceConfigs(false);
}
}
async function pushWorkerGroups(opts: InstanceSyncOptions) {
await pickInstance(opts, true);
const totalChanges = await pushInstanceConfigs(true) ?? 0;
if (totalChanges === 0) {
log.info("No changes to apply");
return;
}
let confirm = true;
if (opts.yes !== true) {
confirm = await Confirm.prompt({
message: `Do you want to apply these ${totalChanges} instance-level changes?`,
default: true,
});
}
if (confirm) {
await pushInstanceConfigs(false);
}
}
const command = new Command()
.description("display worker groups, pull and push worker groups configs")
.action(displayWorkerGroups)
.command("pull")
.description(
"Pull worker groups (similar to `wmill instance pull --skip-users --skip-settings --skip-groups`)"
)
.option(
"--instance",
"Name of the instance to push to, override the active instance"
)
.option(
"--base-url",
"Base url to be passed to the instance settings instead of the local one"
)
.option("--yes", "Pull without needing confirmation")
.action(pullWorkerGroups as any)
.command("push")
.description(
"Push instance settings, users, configs, group and overwrite remote"
)
.option(
"--instance [instance]",
"Name of the instance to push to, override the active instance"
)
.option(
"--base-url [baseUrl]",
"If used with --token, will be used as the base url for the instance"
)
.option("--yes", "Push without needing confirmation")
.action(pushWorkerGroups as any);
export default command;
cli/workers.ts
+105
View File
@@ -0,0 +1,105 @@
import { Command, Table } from "./deps.ts";
import { log } from "./deps.ts";
import * as wmill from "./gen/services.gen.ts";
import { pickInstance } from "./instance.ts";
type GlobalOptions = {
instance?: string;
baseUrl?: string;
};
function toPercent(value: number | undefined): string {
return value != undefined ? `${(value * 100).toFixed(1)}%` : '?%';
}
async function displayWorkers(opts: GlobalOptions) {
const activeInstance = await pickInstance(opts, true);
if (activeInstance) {
const workerGroups = await wmill.listWorkerGroups();
const workers = await wmill.listWorkers({
pingSince: 10
});
const groupedWorkers = workerGroups.map(group => {
const groupWorkers = workers.filter(worker => worker.worker_group === group.name);
return {
groupName: group.name,
workers: groupWorkers
};
});
// Add workers that don't belong to any worker group
const ungroupedWorkers = workers.filter(worker =>
!workerGroups.some(group => group.name === worker.worker_group)
);
if (ungroupedWorkers.length > 0) {
ungroupedWorkers.forEach(worker => {
const groupName = worker.worker_group || "Ungrouped";
let group = groupedWorkers.find(g => g.groupName === groupName);
if (!group) {
group = { groupName, workers: [] };
groupedWorkers.push(group);
}
group.workers.push(worker);
});
}
// Sort groupedWorkers
groupedWorkers.sort((a, b) => {
// Always put 'default' first
if (a.groupName === 'default') return -1;
if (b.groupName === 'default') return 1;
// Then sort by number of workers (descending order)
return b.workers.length - a.workers.length;
});
groupedWorkers.forEach(group => {
log.info(`\nWorker Group: ${group.groupName} (${group.workers.length} workers)`);
if (group.workers.length === 0) {
log.info(" No workers in this group");
} else {
new Table()
.header(["Worker ID", "Host", "Queues", "Jobs", "Occupancy rate 15s/5m/30m/ever)", "Last job", "Last Ping"])
.padding(2)
.border(true)
.maxColWidth(30)
.body(group.workers.map(worker => [
worker.worker,
worker.worker_instance,
worker.custom_tags?.join(', ') || '',
worker.jobs_executed,
`${toPercent(worker.occupancy_rate_15s)}/${toPercent(worker.occupancy_rate_5m)}/${toPercent(worker.occupancy_rate_30m)}/${toPercent(worker.occupancy_rate)}`,
worker.last_job_id ? worker.last_job_id + ' ' +worker.last_job_workspace_id : '',
`${worker.last_ping}s ago`
]))
.render();
}
});
} else {
log.info("No active instance found");
log.info("Use 'wmill instance add' to add a new instance");
}
}
const command = new Command()
.description("List all workers grouped by worker groups")
.option(
"--instance [instance]",
"Name of the instance to push to, override the active instance"
)
.option(
"--base-url [baseUrl]",
"If used with --token, will be used as the base url for the instance"
)
.action(displayWorkers as any);
export default command;
frontend/package-lock.json
Generated
+2 -2
View File
@@ -1,12 +1,12 @@
{
"name": "windmill-components",
"version": "1.401.0",
"version": "1.402.3",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "windmill-components",
"version": "1.401.0",
"version": "1.402.3",
"license": "AGPL-3.0",
"dependencies": {
"@aws-crypto/sha256-js": "^4.0.0",
frontend/package.json
+1 -1
View File
@@ -1,6 +1,6 @@
{
"name": "windmill-components",
"version": "1.401.0",
"version": "1.402.3",
"scripts": {
"dev": "vite dev",
"build": "vite build",
frontend/src/lib/components/DateInput.svelte
+2
View File
@@ -8,6 +8,7 @@
export let minDate: string | undefined = undefined
export let maxDate: string | undefined = undefined
export let dateFormat: string | undefined = 'dd-MM-yyyy'
export let disabled: boolean = false
let date: string | undefined = computeDate(value)
@@ -71,6 +72,7 @@
<div class="flex flex-row gap-1 items-center w-full" id={randomId} on:pointerdown on:focus>
<!-- svelte-ignore a11y-autofocus -->
<input
{disabled}
type="date"
bind:value={date}
{autofocus}
frontend/src/lib/components/EditableSchemaForm.svelte
+1
View File
@@ -448,6 +448,7 @@
{itemPicker}
{lightweightMode}
bind:nullable={schema.properties[argName].nullable}
bind:disabled={schema.properties[argName].disabled}
type={schema.properties[argName].type}
bind:oneOf={schema.properties[argName].oneOf}
bind:format={schema.properties[argName].format}
frontend/src/lib/components/FlowTimeline.svelte
+3 -3
View File
@@ -178,10 +178,10 @@
{/if}
<div class="px-2 py-2 grid grid-cols-6 w-full">
<div>{k}</div>
<div class="col-span-5 flex min-h-6 w-full">
<div class="truncate">{k}</div>
<div class="col-span-5 flex min-h-6">
{#if min && total}
<div class="flex flex-col gap-2 w-full p-2">
<div class="flex flex-col gap-2 w-full p-2 ml-4">
{#each items?.[k] ?? [] as b}
{@const waitingLen = b?.created_at
? b.started_at
frontend/src/lib/components/LightweightArgInput.svelte
+33 -9
View File
@@ -36,6 +36,9 @@
export let type: string | undefined = undefined
export let oneOf: SchemaProperty[] | undefined = undefined
export let required = false
//todo
// export let nullable = false
export let disabled: boolean = false
export let pattern: undefined | string = undefined
export let valid = required ? false : true
export let enum_: EnumType = undefined
@@ -258,7 +261,7 @@
<div class="flex space-x-1">
{#if inputCat == 'number'}
{#if extra['min'] != undefined && extra['max'] != undefined}
<Range bind:value min={extra['min']} max={extra['max']} {defaultValue} />
<Range bind:value min={extra['min']} max={extra['max']} {defaultValue} {disabled} />
{:else if extra?.currency}
<CurrencyInput
inputClasses={{
@@ -270,6 +273,7 @@
bind:value
currency={extra?.currency}
locale={extra?.currencyLocale ?? 'en-US'}
{disabled}
/>
{:else}
<input
@@ -284,6 +288,7 @@
bind:value
min={extra['min']}
max={extra['max']}
{disabled}
/>
{/if}
{:else if inputCat == 'boolean'}
@@ -295,6 +300,7 @@
? ''
: 'border !border-red-700 !border-opacity-70 focus:!border-red-700 focus:!border-opacity-30'}
bind:checked={value}
{disabled}
/>
{#if type == 'boolean' && value == undefined}
<span>&nbsp; Not set</span>
@@ -308,6 +314,7 @@
bind:selected={value}
options={itemsType?.multiselect ?? []}
selectedOptionsDraggable={true}
{disabled}
/>
</div>
{:else if Array.isArray(itemsType?.enum) && Array.isArray(value)}
@@ -317,6 +324,7 @@
bind:selected={value}
options={itemsType?.enum ?? []}
selectedOptionsDraggable={true}
{disabled}
/>
</div>
{:else if Array.isArray(enum_) && Array.isArray(value)}
@@ -326,6 +334,7 @@
bind:selected={value}
options={enum_ ?? []}
selectedOptionsDraggable={true}
{disabled}
/>
</div>
{:else}
@@ -334,13 +343,14 @@
{#each value ?? [] as v, i}
<div class="flex flex-row max-w-md mt-1 w-full">
{#if itemsType?.type == 'number'}
<input type="number" bind:value={v} />
<input type="number" bind:value={v} {disabled} />
{:else if itemsType?.type == 'string' && itemsType?.contentEncoding == 'base64'}
<input
type="file"
class="my-6"
on:change={(x) => fileChanged(x, (val) => (value[i] = val))}
multiple={false}
{disabled}
/>
{:else if Array.isArray(itemsType?.enum)}
<select
@@ -349,6 +359,7 @@
}}
class="px-6"
bind:value={v}
{disabled}
>
{#each itemsType?.enum ?? [] as e}
<option>{e}</option>
@@ -358,16 +369,18 @@
<LightweightResourcePicker
bind:value={v}
resourceType={itemsType?.resourceType}
{disabled}
/>
{:else if itemsType?.type === 'object' && itemsType?.properties}
<div class="p-8 border rounded-md w-full">
<LightweightSchemaForm
schema={getSchemaFromProperties(itemsType?.properties)}
bind:args={v}
{disabled}
/>
</div>
{:else}
<input type="text" bind:value={v} />
<input type="text" bind:value={v} {disabled} />
{/if}
<button
transition:fade|local={{ duration: 100 }}
@@ -379,6 +392,7 @@
value = undefined
}
}}
{disabled}
>
<X size={14} />
</button>
@@ -398,6 +412,7 @@
addItemByItemsType()
}}
startIcon={{ icon: Plus }}
{disabled}
>
Add
</Button>
@@ -430,13 +445,13 @@
/>
</div>
{:else if inputCat == 'resource-object'}
<LightweightObjectResourceInput {format} bind:value />
<LightweightObjectResourceInput {format} bind:value {disabled} />
{:else if inputCat == 'object' && oneOf && oneOf.length >= 2}
<div class="flex flex-col gap-2 w-full">
{#if oneOf && oneOf.length >= 2}
<ToggleButtonGroup bind:selected={oneOfSelected}>
<ToggleButtonGroup bind:selected={oneOfSelected} {disabled}>
{#each oneOf as obj}
<ToggleButton value={obj.title} label={obj.title} />
<ToggleButton value={obj.title} label={obj.title} {disabled} />
{/each}
</ToggleButtonGroup>
@@ -455,6 +470,7 @@
type: 'object'
}}
bind:args={value}
{disabled}
/>
</div>
{/if}
@@ -472,6 +488,7 @@
type: 'object'
}}
bind:args={value}
{disabled}
/>
</div>
{:else}
@@ -487,6 +504,7 @@
: 'border !border-red-700 !border-opacity-70 focus:!border-red-700 focus:!border-opacity-30'}"
placeholder={defaultValue ? JSON.stringify(defaultValue, null, 4) : ''}
bind:value={rawValue}
{disabled}
/>
{/if}
{:else if inputCat == 'enum'}
@@ -496,6 +514,7 @@
}}
class="px-6"
bind:value
{disabled}
>
{#each enum_ ?? [] as e}
<option value={e}>{extra?.['enumLabels']?.[e] ?? e}</option>
@@ -504,9 +523,9 @@
{:else if inputCat == 'date'}
{#key defaultChange}
{#if format === 'date'}
<DateInput bind:value dateFormat={extra['dateFormat']} />
<DateInput bind:value dateFormat={extra['dateFormat']} {disabled} />
{:else}
<DateTimeInput useDropdown bind:value />
<DateTimeInput useDropdown bind:value {disabled} />
{/if}
{/key}
{:else if inputCat == 'base64'}
@@ -515,6 +534,7 @@
type="file"
on:change={(x) => fileChanged(x, (val) => (value = val))}
multiple={false}
{disabled}
/>
{#if value?.length}
<div class="text-2xs text-tertiary mt-1"
@@ -531,6 +551,7 @@
resourceType={format.split('-').length > 1
? format.substring('resource-'.length)
: undefined}
{disabled}
/>
</div>
{:else if inputCat == 'email'}
@@ -542,6 +563,7 @@
: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-3'}
placeholder={placeholder ?? defaultValue ?? ''}
bind:value
{disabled}
/>
{:else if inputCat == 'currency'}
<input
@@ -551,12 +573,13 @@
: 'border border-red-700 border-opacity-30 focus:border-red-700 focus:border-opacity-3'}
placeholder={placeholder ?? defaultValue ?? ''}
bind:value
{disabled}
/>
{:else if inputCat == 'string'}
<div class="flex flex-col w-full">
<div class="flex flex-row w-full items-center justify-between">
{#if extra?.['password'] == true}
<Password bind:password={value} />
<Password bind:password={value} {disabled} />
{:else}
<textarea
rows={extra?.['minRows'] || 1}
@@ -573,6 +596,7 @@
on:pointerdown|stopPropagation={(e) => {
dispatch('inputClicked', e)
}}
{disabled}
/>
{/if}
</div>
frontend/src/lib/components/LightweightObjectResourceInput.svelte
+2 -1
View File
@@ -4,7 +4,7 @@
export let format: string
export let value: any
export let disablePortal = false
export let disabled: boolean = false
function isString(value: any) {
return typeof value === 'string' || value instanceof String
}
@@ -34,6 +34,7 @@
<div class="flex flex-row w-full flex-wrap gap-x-2 gap-y-0.5">
<LightweightResourcePicker
{disabled}
{disablePortal}
on:change={(e) => {
path = e.detail
frontend/src/lib/components/LightweightSchemaForm.svelte
+2
View File
@@ -15,6 +15,7 @@
export let isValid: boolean = true
export let defaultValues: Record<string, any> = {}
export let dynamicEnums: Record<string, any> = {}
export let disabled: boolean = false
let inputCheck: { [id: string]: boolean } = {}
let errors: { [id: string]: string } = {}
@@ -102,6 +103,7 @@
on:inputClicked
{displayType}
{css}
disabled={disabled || schema.properties[argName].disabled}
/>
{/if}
{/each}
frontend/src/lib/components/OAuthSetting.svelte
+39 -15
View File
@@ -1,4 +1,5 @@
<script lang="ts">
import { X } from 'lucide-svelte'
import CollapseLink from './CollapseLink.svelte'
import IconedResourceType from './IconedResourceType.svelte'
import Toggle from './Toggle.svelte'
@@ -9,8 +10,6 @@
$: enabled = value != undefined
let allowed_domains = value?.['allowed_domains'] ?? ''
let tenant: string = ''
$: name == 'microsoft' && changeTenantId(tenant)
@@ -66,19 +65,44 @@
</label>
{:else if login}
<label class="block pb-2">
<span class="text-primary font-semibold text-sm">Allowed domain</span>
<input
type="text"
placeholder="windmill.dev"
bind:value={allowed_domains}
on:keyup={() => {
if (allowed_domains == '') {
value['allowed_domains'] = undefined
} else {
value['allowed_domains'] = [allowed_domains]
}
}}
/>
<span class="text-primary font-semibold text-sm">Allowed domains</span>
<div class="flex flex-col gap-1">
{#each value?.['allowed_domains'] ?? [] as domain}
<div class="flex gap-2">
<input
class="max-w-96 w-full"
type="text"
bind:value={domain}
on:keyup={(e) => {
if (domain == '') {
value['allowed_domains'] = value['allowed_domains']?.filter(
(d) => d != domain
)
}
}}
/>
<button
class="text-primary text-xs rounded hover:bg-surface-hover"
on:click={() => {
value['allowed_domains'] = value['allowed_domains']?.filter((d) => d != domain)
if (value['allowed_domains'].length == 0) {
value['allowed_domains'] = undefined
}
}}
>
<X size={14} />
</button>
</div>
{/each}
<div class="flex gap-2">
<button
class="text-primary text-sm border rounded p-1"
on:click={() => {
value['allowed_domains'] = [...(value['allowed_domains'] ?? []), 'mydomain.com']
}}>+ Add domain</button
>
</div>
</div>
</label>
{/if}
{#if name == 'google'}
frontend/src/lib/components/SchemaForm.svelte
+6 -1
View File
@@ -184,6 +184,9 @@
extra={schema.properties[argName]}
title={schema.properties[argName].title}
placeholder={schema.properties[argName].placeholder}
disabled={disabledArgs.includes(argName) ||
disabled ||
schema.properties[argName].disabled}
>
<svelte:fragment slot="actions">
<slot name="actions" />
@@ -240,7 +243,9 @@
bind:order={schema.properties[argName].order}
nestedRequired={schema.properties[argName]?.required}
itemsType={schema.properties[argName].items}
disabled={disabledArgs.includes(argName) || disabled}
disabled={disabledArgs.includes(argName) ||
disabled ||
schema.properties[argName].disabled}
{compact}
{variableEditor}
{itemPicker}
frontend/src/lib/components/common/menu/MenuV2.svelte
+3 -2
View File
@@ -10,7 +10,7 @@
export let placement: any = 'bottom-start'
export let justifyEnd: boolean = false
export let lightMode: boolean = false
export let maxHeight: number = 900
const [floatingRef, floatingContent] = createFloatingActions({
strategy: 'fixed',
middleware: [offset(), flip(), shift()],
@@ -38,9 +38,10 @@
>
<MenuItems
class={twMerge(
'border w-56 origin-top-right rounded-md shadow-md focus:outline-none',
'border w-56 origin-top-right rounded-md shadow-md focus:outline-none overflow-y-auto',
lightMode ? 'bg-surface-inverse' : 'bg-surface'
)}
style="max-height: {maxHeight}px;"
>
<div class="my-1">
<slot />
frontend/src/lib/components/flows/map/FlowJobsMenu.svelte
+1 -1
View File
@@ -26,7 +26,7 @@
}
</script>
<Menu>
<Menu maxHeight={300}>
<div slot="trigger">
<button
title="Pick an iteration"
frontend/src/lib/components/flows/map/FlowModuleSchemaItem.svelte
+9 -10
View File
@@ -224,13 +224,12 @@
<slot name="icon" />
{/if}
<div class="truncate" class:font-bold={bold}>{label}</div>
<div class="flex items-center space-x-2">
<div class="flex items-center space-x-2 relative">
{#if id && id !== 'preprocessor' && !id.startsWith('failure')}
<Badge color="indigo">{id}</Badge>
{#if deletable}
<button
class="absolute -left-[20px] z-10 h-[20px] rounded-l rounded-t rounded-s w-[20px] trash center-center text-secondary bg-surface duration-150 hover:bg-blue-400 {editId
class="absolute -left-[28px] z-10 h-[20px] rounded-l rounded-t rounded-s w-[20px] trash center-center text-secondary bg-surface duration-150 hover:bg-blue-400 {editId
? '!bg-blue-400'
: ''} hover:text-white
hover:border-blue-700 hover:!visible {hover ? '' : '!hidden'}"
@@ -254,15 +253,15 @@ hover:border-blue-700 hover:!visible {hover ? '' : '!hidden'}"
</button>
{#if id !== 'preprocessor'}
<button
class="absolute -top-[10px] right-[60px] rounded-full h-[20px] w-[20px] trash center-center text-secondary
<button
class="absolute -top-[10px] right-[60px] rounded-full h-[20px] w-[20px] trash center-center text-secondary
outline-[1px] outline dark:outline-gray-500 outline-gray-300 bg-surface duration-150 hover:bg-blue-400 hover:text-white
{hover ? '' : '!hidden'}"
on:click|preventDefault|stopPropagation={(event) => dispatch('move')}
title="Move"
>
<Move class="mx-[3px]" size={12} strokeWidth={2} />
</button>
on:click|preventDefault|stopPropagation={(event) => dispatch('move')}
title="Move"
>
<Move class="mx-[3px]" size={12} strokeWidth={2} />
</button>
{/if}
{#if (id && Object.values($flowInputsStore?.[id]?.flowStepWarnings || {}).length > 0) || Boolean(warningMessage)}
frontend/src/lib/components/flows/map/FlowModuleSchemaMap.svelte
+1 -1
View File
@@ -254,7 +254,7 @@
{/if}
</div>
<div class="z-10 flex-auto grow" bind:clientHeight={minHeight}>
<div class="z-10 flex-auto grow bg-surface-secondary" bind:clientHeight={minHeight}>
<FlowGraphV2
path={$pathStore}
isEditor={true}
frontend/src/lib/components/graph/FlowGraphV2.svelte
+1
View File
@@ -299,6 +299,7 @@
elementsSelectable={false}
{proOptions}
nodesDraggable={false}
--background-color={false}
>
<div class="absolute inset-0 !bg-surface-secondary" />
<Controls position="top-right" orientation="horizontal" showLock={false}>
frontend/src/lib/components/schema/FlowPropertyEditor.svelte
+12
View File
@@ -30,6 +30,7 @@
export let variableEditor: VariableEditor | undefined = undefined
export let itemPicker: ItemPicker | undefined = undefined
export let nullable: boolean = false
export let disabled: boolean = false
export let defaultValue: any = undefined
export let propsNames: any = []
export let showExpr: string | undefined = undefined
@@ -336,6 +337,7 @@
{contentEncoding}
{format}
{extra}
{disabled}
/>
{:else}
<ArgInput
@@ -350,6 +352,7 @@
{contentEncoding}
{format}
{extra}
{disabled}
{nullable}
{variableEditor}
compact
@@ -386,6 +389,15 @@
disabled={required}
/>
{/if}
<Toggle
options={{
right: 'Disabled',
rightTooltip: 'Do not let user modify this field'
}}
lightMode
size="xs"
bind:checked={disabled}
/>
</div>
{#if displayWebhookWarning && !(type === 'object' && oneOf && oneOf.length >= 2)}
frontend/src/lib/components/sidebar/changelogs.ts
+10
View File
@@ -5,6 +5,16 @@ export type Changelog = {
}
const changelogs: Changelog[] = [
{
label: 'Custom HTTP routes',
href: 'https://www.windmill.dev/changelog/http-routing',
date: '2024-09-23'
},
{
label: 'Set/Get progress from code',
href: 'https://www.windmill.dev/changelog/explicit-progress',
date: '2024-09-18'
},
{
label: 'Directly edit flow YAML',
href: 'https://www.windmill.dev/changelog/flow-yaml-editor',
frontend/src/routes/(root)/(logged)/workers/+page.svelte
+16 -4
View File
@@ -74,7 +74,7 @@
async function loadWorkerGroups(): Promise<void> {
try {
workerGroups = Object.fromEntries(
(await ConfigService.listWorkerGroups()).map((x) => [x.name.substring(8), x.config])
(await ConfigService.listWorkerGroups()).map((x) => [x.name, x.config])
)
} catch (err) {
sendUserToast(`Could not load worker groups: ${err}`, true)
@@ -240,6 +240,14 @@
const openSearchWithPrefilledText: (t?: string) => void = getContext(
'openSearchWithPrefilledText'
)
function displayOccupancyRate(occupancy_rate: number | undefined) {
if (occupancy_rate == undefined) {
return '--'
}
return Math.ceil(occupancy_rate * 100) + '%'
}
</script>
{#if $superadmin}
@@ -503,7 +511,7 @@
<Cell head>Jobs ran</Cell>
{#if (!config || config?.dedicated_worker == undefined) && $superadmin}
<Cell head>Last job</Cell>
<Cell head>Occupancy rate</Cell>
<Cell head>Occupancy rate<br />(15s/5m/30m/ever)</Cell>
{/if}
<Cell head>Memory usage<br />(Windmill)</Cell>
<Cell head>Limits</Cell>
@@ -538,7 +546,7 @@
</tr>
{#if workers}
{#each workers as { worker, custom_tags, last_ping, started_at, jobs_executed, last_job_id, last_job_workspace_id, occupancy_rate, wm_version, vcpus, memory, memory_usage, wm_memory_usage }}
{#each workers as { worker, custom_tags, last_ping, started_at, jobs_executed, last_job_id, last_job_workspace_id, occupancy_rate_15s, occupancy_rate_5m, occupancy_rate_30m, occupancy_rate, wm_version, vcpus, memory, memory_usage, wm_memory_usage }}
<tr>
<Cell first>{worker}</Cell>
<Cell>
@@ -564,7 +572,11 @@
{/if}
</Cell>
<Cell>
{Math.ceil((occupancy_rate ?? 0) * 100)}%
{displayOccupancyRate(occupancy_rate_15s)}/{displayOccupancyRate(
occupancy_rate_5m
)}/{displayOccupancyRate(occupancy_rate_30m)}/{displayOccupancyRate(
occupancy_rate
)}
</Cell>
{/if}
<Cell>
lsp/Pipfile
+2 -2
View File
@@ -4,8 +4,8 @@ verify_ssl = true
name = "pypi"
[packages]
wmill = ">=1.401.0"
wmill_pg = ">=1.401.0"
wmill = ">=1.402.3"
wmill_pg = ">=1.402.3"
sendgrid = "*"
mysql-connector-python = "*"
pymongo = "*"
openflow.openapi.yaml
+1 -1
View File
@@ -1,7 +1,7 @@
openapi: "3.0.3"
info:
version: 1.401.0
version: 1.402.3
title: OpenFlow Spec
contact:
name: Ruben Fiszel

Some files were not shown because too many files have changed in this diff Show More