From bba0091bfff717e40b6fbfd6e979ab6e928eb39f Mon Sep 17 00:00:00 2001 From: HugoCasa Date: Wed, 29 May 2024 01:12:00 +0200 Subject: [PATCH] fix: worker metrics (#3837) * fix: worker metrics * chore: sqlx * chore: update ee ref * fix: remove jemalloc stats unwrap --- ...1a70938a3ecd7ff8a89277cc35d119e2bad4.json} | 5 +- ...485b8ccb2d07d723f184cef2696760d3ccf9.json} | 10 +- ...d7378b0c9d3b5f3d6bd6cafcba273f8277d4.json} | 6 +- backend/Cargo.lock | 3 + backend/Cargo.toml | 14 ++- backend/ee-repo-ref.txt | 2 +- ...8135719_add_windmill_memory_usage.down.sql | 1 + ...528135719_add_windmill_memory_usage.up.sql | 2 + backend/src/monitor.rs | 42 +++++-- backend/windmill-api/openapi.yaml | 2 + backend/windmill-api/src/workers.rs | 4 +- backend/windmill-common/Cargo.toml | 5 + backend/windmill-common/src/worker.rs | 119 +++++++++++------- backend/windmill-worker/src/common.rs | 10 +- backend/windmill-worker/src/worker.rs | 7 +- .../(root)/(logged)/workers/+page.svelte | 25 ++-- 16 files changed, 176 insertions(+), 81 deletions(-) rename backend/.sqlx/{query-177661a6487cfef198c2ceca4c81e546230c2fd7756d13197a7c19a78eb4b3d6.json => query-4398bc6f701dcba647a259ec2d091a70938a3ecd7ff8a89277cc35d119e2bad4.json} (64%) rename backend/.sqlx/{query-99d79e2a3792296b0582bbaf8aa1d39855d2109578d0a62d7f2cf1444e44550e.json => query-55d6c6f887f1cceb311d9ab034d3485b8ccb2d07d723f184cef2696760d3ccf9.json} (84%) rename backend/.sqlx/{query-8c62e3bb264c7336b3b3a68677993ba3b729c7358cc3da14513d8b447ce65aaf.json => query-95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4.json} (54%) create mode 100644 backend/migrations/20240528135719_add_windmill_memory_usage.down.sql create mode 100644 backend/migrations/20240528135719_add_windmill_memory_usage.up.sql diff --git a/backend/.sqlx/query-177661a6487cfef198c2ceca4c81e546230c2fd7756d13197a7c19a78eb4b3d6.json b/backend/.sqlx/query-4398bc6f701dcba647a259ec2d091a70938a3ecd7ff8a89277cc35d119e2bad4.json similarity index 64% rename from backend/.sqlx/query-177661a6487cfef198c2ceca4c81e546230c2fd7756d13197a7c19a78eb4b3d6.json rename to backend/.sqlx/query-4398bc6f701dcba647a259ec2d091a70938a3ecd7ff8a89277cc35d119e2bad4.json index 4a1be7e7b4..594ad1092f 100644 --- a/backend/.sqlx/query-177661a6487cfef198c2ceca4c81e546230c2fd7756d13197a7c19a78eb4b3d6.json +++ b/backend/.sqlx/query-4398bc6f701dcba647a259ec2d091a70938a3ecd7ff8a89277cc35d119e2bad4.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, current_job_id = NULL, current_job_workspace_id = NULL WHERE worker = $5", + "query": "UPDATE worker_ping SET ping_at = now(), jobs_executed = $1, custom_tags = $2, occupancy_rate = $3, memory_usage = $4, wm_memory_usage = $5, current_job_id = NULL, current_job_workspace_id = NULL WHERE worker = $6", "describe": { "columns": [], "parameters": { @@ -9,10 +9,11 @@ "TextArray", "Float4", "Int8", + "Int8", "Text" ] }, "nullable": [] }, - "hash": "177661a6487cfef198c2ceca4c81e546230c2fd7756d13197a7c19a78eb4b3d6" + "hash": "4398bc6f701dcba647a259ec2d091a70938a3ecd7ff8a89277cc35d119e2bad4" } diff --git a/backend/.sqlx/query-99d79e2a3792296b0582bbaf8aa1d39855d2109578d0a62d7f2cf1444e44550e.json b/backend/.sqlx/query-55d6c6f887f1cceb311d9ab034d3485b8ccb2d07d723f184cef2696760d3ccf9.json similarity index 84% rename from backend/.sqlx/query-99d79e2a3792296b0582bbaf8aa1d39855d2109578d0a62d7f2cf1444e44550e.json rename to backend/.sqlx/query-55d6c6f887f1cceb311d9ab034d3485b8ccb2d07d723f184cef2696760d3ccf9.json index 553cf6494e..dc06930fb1 100644 --- a/backend/.sqlx/query-99d79e2a3792296b0582bbaf8aa1d39855d2109578d0a62d7f2cf1444e44550e.json +++ b/backend/.sqlx/query-55d6c6f887f1cceb311d9ab034d3485b8ccb2d07d723f184cef2696760d3ccf9.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as current_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as current_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", + "query": "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as current_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as current_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage\n FROM worker_ping\n WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval)\n ORDER BY ping_at desc LIMIT $2 OFFSET $3", "describe": { "columns": [ { @@ -77,6 +77,11 @@ "ordinal": 14, "name": "memory_usage", "type_info": "Int8" + }, + { + "ordinal": 15, + "name": "wm_memory_usage", + "type_info": "Int8" } ], "parameters": { @@ -102,8 +107,9 @@ true, true, true, + true, true ] }, - "hash": "99d79e2a3792296b0582bbaf8aa1d39855d2109578d0a62d7f2cf1444e44550e" + "hash": "55d6c6f887f1cceb311d9ab034d3485b8ccb2d07d723f184cef2696760d3ccf9" } diff --git a/backend/.sqlx/query-8c62e3bb264c7336b3b3a68677993ba3b729c7358cc3da14513d8b447ce65aaf.json b/backend/.sqlx/query-95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4.json similarity index 54% rename from backend/.sqlx/query-8c62e3bb264c7336b3b3a68677993ba3b729c7358cc3da14513d8b447ce65aaf.json rename to backend/.sqlx/query-95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4.json index e4a08130cf..991be7c977 100644 --- a/backend/.sqlx/query-8c62e3bb264c7336b3b3a68677993ba3b729c7358cc3da14513d8b447ce65aaf.json +++ b/backend/.sqlx/query-95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4.json @@ -1,16 +1,18 @@ { "db_name": "PostgreSQL", - "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2 WHERE worker = $3", + "query": "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4 WHERE worker = $5", "describe": { "columns": [], "parameters": { "Left": [ "Uuid", "Varchar", + "Int8", + "Int8", "Text" ] }, "nullable": [] }, - "hash": "8c62e3bb264c7336b3b3a68677993ba3b729c7358cc3da14513d8b447ce65aaf" + "hash": "95cb1fe8658f98fb736d899fa21cd7378b0c9d3b5f3d6bd6cafcba273f8277d4" } diff --git a/backend/Cargo.lock b/backend/Cargo.lock index aa452f8898..8a74c11fef 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -8313,6 +8313,7 @@ dependencies = [ "libc", "ntapi", "once_cell", + "rayon", "windows", ] @@ -9872,7 +9873,9 @@ dependencies = [ "serde_json", "sha2 0.10.8", "sqlx", + "sysinfo", "thiserror", + "tikv-jemalloc-ctl", "tokio", "tracing", "tracing-flame", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index bceb452909..ff3073e5e3 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -51,7 +51,7 @@ prometheus = ["windmill-common/prometheus", "windmill-api/prometheus", "windmill flow_testing = ["windmill-worker/flow_testing"] openidconnect = ["windmill-api/openidconnect"] cloud = ["windmill-queue/cloud"] -jemalloc = ["dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] +jemalloc = ["windmill-common/jemalloc", "dep:tikv-jemallocator", "dep:tikv-jemalloc-sys", "dep:tikv-jemalloc-ctl"] [dependencies] anyhow.workspace = true @@ -84,9 +84,9 @@ deno_core.workspace = true pg-embed = {git = "https://github.com/faokunega/pg-embed", optional = true, default-features = false, features = ['rt_tokio']} [target.'cfg(not(target_env = "msvc"))'.dependencies] -tikv-jemallocator = { optional = true, version = "0.5", features = ["profiling"] } -tikv-jemalloc-sys = { optional = true, version = "^0.5", features = ["profiling"] } -tikv-jemalloc-ctl = { optional = true, version = "^0.5" } +tikv-jemallocator = { optional = true, workspace = true } +tikv-jemalloc-sys = { optional = true, workspace = true } +tikv-jemalloc-ctl = { optional = true, workspace = true } [dev-dependencies] serde_json.workspace = true @@ -248,3 +248,9 @@ crc = "^3" tar = "^0" http = "^1" async-stream = "^0" + +sysinfo = "0.30.12" + +tikv-jemallocator = { version = "0.5", features = ["profiling"] } +tikv-jemalloc-sys = { version = "^0.5", features = ["profiling"] } +tikv-jemalloc-ctl = { version = "^0.5" } diff --git a/backend/ee-repo-ref.txt b/backend/ee-repo-ref.txt index 71d150bec3..3f723302af 100644 --- a/backend/ee-repo-ref.txt +++ b/backend/ee-repo-ref.txt @@ -1 +1 @@ -cea722235664845af25689abf98c5fd2678ddf76 \ No newline at end of file +553aa1d75498c9c6c07a1495a154478fb0bc064e \ No newline at end of file diff --git a/backend/migrations/20240528135719_add_windmill_memory_usage.down.sql b/backend/migrations/20240528135719_add_windmill_memory_usage.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20240528135719_add_windmill_memory_usage.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20240528135719_add_windmill_memory_usage.up.sql b/backend/migrations/20240528135719_add_windmill_memory_usage.up.sql new file mode 100644 index 0000000000..7c2314e49c --- /dev/null +++ b/backend/migrations/20240528135719_add_windmill_memory_usage.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TABLE worker_ping ADD COLUMN wm_memory_usage BIGINT; \ No newline at end of file diff --git a/backend/src/monitor.rs b/backend/src/monitor.rs index fb0d0513c5..4e0affaa9c 100644 --- a/backend/src/monitor.rs +++ b/backend/src/monitor.rs @@ -250,19 +250,43 @@ pub async fn monitor_mem() { // Obtain a MIB for the `epoch`, `stats.allocated`, and // `atats.resident` keys: - let e = epoch::mib().unwrap(); - let allocated = stats::allocated::mib().unwrap(); - let resident = stats::resident::mib().unwrap(); + let e = match epoch::mib() { + Ok(mib) => mib, + Err(e) => { + tracing::error!("Error getting jemalloc epoch mib: {:?}", e); + return; + } + }; + let allocated = match stats::allocated::mib() { + Ok(mib) => mib, + Err(e) => { + tracing::error!("Error getting jemalloc allocated mib: {:?}", e); + return; + } + }; + let resident = match stats::resident::mib() { + Ok(mib) => mib, + Err(e) => { + tracing::error!("Error getting jemalloc resident mib: {:?}", e); + return; + } + }; + loop { // Many statistics are cached and only updated // when the epoch is advanced: - e.advance().unwrap(); - - // Read statistics using MIB key: - let allocated = allocated.read().unwrap(); - let resident = resident.read().unwrap(); - tracing::info!("{} mb allocated/{} mb resident", bytes_to_mb(allocated as u64), bytes_to_mb(resident as u64)); + match e.advance() { + Ok(_) => { + // Read statistics using MIB key: + let allocated = allocated.read().unwrap_or_default(); + let resident = resident.read().unwrap_or_default(); + tracing::info!("{} mb allocated/{} mb resident", bytes_to_mb(allocated as u64), bytes_to_mb(resident as u64)); + }, + Err(e) => { + tracing::error!("Error advancing jemalloc epoch: {:?}", e); + } + } tokio::time::sleep(Duration::from_secs(10)).await; } }); diff --git a/backend/windmill-api/openapi.yaml b/backend/windmill-api/openapi.yaml index 8e0990e065..1a449be377 100644 --- a/backend/windmill-api/openapi.yaml +++ b/backend/windmill-api/openapi.yaml @@ -9913,6 +9913,8 @@ components: type: number memory_usage: type: number + wm_memory_usage: + type: number required: - worker - worker_instance diff --git a/backend/windmill-api/src/workers.rs b/backend/windmill-api/src/workers.rs index 71c5be4dfd..17fcbe0704 100644 --- a/backend/windmill-api/src/workers.rs +++ b/backend/windmill-api/src/workers.rs @@ -59,6 +59,8 @@ struct WorkerPing { vcpus: Option, #[serde(skip_serializing_if = "Option::is_none")] memory_usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + wm_memory_usage: Option, } #[derive(Serialize, Deserialize)] @@ -86,7 +88,7 @@ async fn list_worker_pings( let rows = sqlx::query_as!( WorkerPing, - "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as current_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as current_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage + "SELECT worker, worker_instance, EXTRACT(EPOCH FROM (now() - ping_at))::integer as last_ping, started_at, ip, jobs_executed, CASE WHEN $4 IS TRUE THEN current_job_id ELSE NULL END as current_job_id, CASE WHEN $4 IS TRUE THEN current_job_workspace_id ELSE NULL END as current_job_workspace_id, custom_tags, worker_group, wm_version, occupancy_rate, memory, vcpus, memory_usage, wm_memory_usage FROM worker_ping WHERE ($1::integer IS NULL AND ping_at > now() - interval '5 minute') OR (ping_at > now() - ($1 || ' seconds')::interval) ORDER BY ping_at desc LIMIT $2 OFFSET $3", diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml index e1c0af06fa..deab5c347e 100644 --- a/backend/windmill-common/Cargo.toml +++ b/backend/windmill-common/Cargo.toml @@ -7,6 +7,7 @@ edition.workspace = true [features] default = [] enterprise = [] +jemalloc = ["dep:tikv-jemalloc-ctl"] prometheus = ["dep:prometheus"] flamegraph = ["dep:tracing-flame"] loki = ["dep:tracing-loki"] @@ -49,3 +50,7 @@ aws-sdk-sts = { workspace = true, optional = true } indexmap.workspace = true bytes = { workspace = true, optional = true } mail-send.workspace = true +sysinfo.workspace = true + +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemalloc-ctl = { optional = true, workspace = true } diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index bee9fb83ec..52c2129929 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -7,6 +7,7 @@ use std::{ collections::{HashMap, HashSet}, sync::{atomic::AtomicBool, Arc}, }; +use sysinfo::{MemoryRefreshKind, System}; use tokio::sync::RwLock; use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::ServerConfig, DB}; @@ -132,22 +133,81 @@ fn process_custom_tags(tags: Vec) -> (Vec, HashMap Option { - let mut memory = std::process::Command::new("cat") - .args(["/sys/fs/cgroup/memory.current"]) +pub fn get_vcpus() -> Option { + let mut vcpus = std::process::Command::new("cat") + .args(["/sys/fs/cgroup/cpu.max"]) .output() .ok() - .map(|o| String::from_utf8_lossy(&o.stdout).to_string()); + .map(|o| { + String::from_utf8_lossy(&o.stdout) + .to_string() + .split(" ") + .map(|s| s.to_string()) + .collect::>() + .get(0) + .map(|s| s.to_string().trim().parse::().ok()) + .flatten() + }) + .flatten(); - if memory.is_none() { - memory = std::process::Command::new("cat") - .args(["/sys/fs/cgroup/memory/memory.usage_in_bytes"]) + if vcpus.is_none() { + vcpus = std::process::Command::new("cat") + .args(["/sys/fs/cgroup/cpu/cpu.cfs_quota_us"]) .output() .ok() - .map(|o| String::from_utf8_lossy(&o.stdout).to_string()) + .map(|o| { + String::from_utf8_lossy(&o.stdout) + .to_string() + .trim() + .parse::() + .ok() + }) + .flatten() } - memory.map(|x| x.parse::().ok()).flatten() + vcpus +} + +pub fn get_memory() -> Option { + let mut sys = System::new(); + sys.refresh_memory(); + let limits = sys.cgroup_limits(); + + if let Some(limits) = limits { + i64::try_from(limits.total_memory).ok() + } else { + None + } +} + +pub fn get_worker_memory_usage() -> Option { + let mut sys = System::new(); + sys.refresh_memory_specifics(MemoryRefreshKind::new().with_ram()); + i64::try_from(sys.used_memory()).ok() +} + +pub fn get_windmill_memory_usage() -> Option { + #[cfg(all(not(target_env = "msvc"), feature = "jemalloc"))] + { + match tikv_jemalloc_ctl::epoch::advance() { + Ok(_) => match tikv_jemalloc_ctl::stats::resident::read() { + Ok(resident) => i64::try_from(resident).ok(), + Err(e) => { + tracing::error!("jemalloc resident memory read failed: {:?}", e); + None + } + }, + Err(e) => { + tracing::error!("jemalloc epoch advance failed: {:?}", e); + None + } + } + } + + #[cfg(any(target_env = "msvc", not(feature = "jemalloc")))] + { + None + } } pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) { @@ -160,42 +220,9 @@ pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: .map(|x| format!("{}:{}", x.workspace_id, x.path)), ) }; - let mut vcpus = std::process::Command::new("cat") - .args(["/sys/fs/cgroup/cpu.max"]) - .output() - .ok() - .map(|o| { - String::from_utf8_lossy(&o.stdout) - .to_string() - .split(" ") - .map(|s| s.to_string()) - .collect::>() - .get(0) - .map(|s| s.to_string()) - }) - .flatten(); - if vcpus.is_none() { - vcpus = std::process::Command::new("cat") - .args(["/sys/fs/cgroup/cpu/cpu.cfs_quota_us"]) - .output() - .ok() - .map(|o| String::from_utf8_lossy(&o.stdout).to_string()) - } - - let mut memory = std::process::Command::new("cat") - .args(["/sys/fs/cgroup/memory.max"]) - .output() - .ok() - .map(|o| String::from_utf8_lossy(&o.stdout).to_string()); - - if memory.is_none() { - memory = std::process::Command::new("cat") - .args(["/sys/fs/cgroup/memory/memory.limit_in_bytes"]) - .output() - .ok() - .map(|o| String::from_utf8_lossy(&o.stdout).to_string()) - } + let vcpus = get_vcpus(); + let memory = get_memory(); sqlx::query!( "INSERT INTO worker_ping (worker_instance, worker, ip, custom_tags, worker_group, dedicated_worker, wm_version, vcpus, memory) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) ON CONFLICT (worker) DO UPDATE set ip = $3, custom_tags = $4, worker_group = $5", @@ -206,8 +233,8 @@ pub async fn update_ping(worker_instance: &str, worker_name: &str, ip: &str, db: *WORKER_GROUP, dw, crate::utils::GIT_VERSION, - vcpus.map(|x| x.parse::().ok()).flatten(), - memory.map(|x| x.parse::().ok()).flatten() + vcpus, + memory ) .execute(db) .await diff --git a/backend/windmill-worker/src/common.rs b/backend/windmill-worker/src/common.rs index fe73ef0116..47aeb3573c 100644 --- a/backend/windmill-worker/src/common.rs +++ b/backend/windmill-worker/src/common.rs @@ -27,7 +27,9 @@ use windmill_common::s3_helpers::{ get_etag_or_empty, LargeFileStorage, ObjectStoreResource, S3Object, }; use windmill_common::variables::{build_crypt_with_key_suffix, decrypt_value_with_mc}; -use windmill_common::worker::{CLOUD_HOSTED, TMP_DIR, WORKER_CONFIG}; +use windmill_common::worker::{ + get_windmill_memory_usage, get_worker_memory_usage, CLOUD_HOSTED, TMP_DIR, WORKER_CONFIG, +}; use windmill_common::{ error::{self, Error}, jobs::QueuedJob, @@ -582,10 +584,14 @@ where // update the last_ping column every 5 seconds i+=1; if i % 10 == 0 { + let memory_usage = get_worker_memory_usage(); + let wm_memory_usage = get_windmill_memory_usage(); sqlx::query!( - "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2 WHERE worker = $3", + "UPDATE worker_ping SET ping_at = now(), current_job_id = $1, current_job_workspace_id = $2, memory_usage = $3, wm_memory_usage = $4 WHERE worker = $5", &job_id, &w_id, + memory_usage, + wm_memory_usage, &worker_name ) .execute(&db) diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 0e0697add9..b17dcea4b9 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -6,7 +6,7 @@ * LICENSE-AGPL for a copy of the license. */ -use windmill_common::worker::{get_worker_memory_usage, TMP_DIR}; +use windmill_common::worker::{get_windmill_memory_usage, get_worker_memory_usage, TMP_DIR}; use anyhow::Result; use const_format::concatcp; @@ -1427,13 +1427,14 @@ pub async fn run_worker {#each groupedWorkers as worker_group (worker_group[0])} {@const config = (workerGroups ?? {})[worker_group[0]]} + {@const activeWorkers = worker_group?.[1].flatMap((x) => + x[1]?.filter((y) => (y.last_ping ?? 0) < 15) + )} { loadWorkerGroups() }} - activeWorkers={worker_group?.[1].flatMap((x) => - x[1]?.filter((y) => (y.last_ping ?? 0) < 15) - )?.length ?? 0} + activeWorkers={activeWorkers?.length ?? 0} {defaultTagPerWorkspace} /> @@ -340,13 +341,13 @@ Last ping Worker start - Nb of jobs executed + Jobs ran {#if (!config || config?.dedicated_worker == undefined) && $superadmin} Current job Occupancy rate {/if} - Memory usage - vCPUs/memory limits + Memory usage
(Windmill usage) + Limits Version Liveness @@ -369,7 +370,7 @@ {#if workers} - {#each workers as { worker, custom_tags, last_ping, started_at, jobs_executed, current_job_id, current_job_workspace_id, occupancy_rate, wm_version, vcpus, memory, memory_usage }} + {#each workers as { worker, custom_tags, last_ping, started_at, jobs_executed, current_job_id, current_job_workspace_id, occupancy_rate, wm_version, vcpus, memory, memory_usage, wm_memory_usage }} {worker} @@ -389,6 +390,7 @@ View job +
(workspace {current_job_workspace_id}) {/if} @@ -396,10 +398,15 @@ {Math.ceil(occupancy_rate ?? 0 * 100)}% {/if} - {memory_usage ? Math.round(memory_usage / 1000000) + 'MB' : '--'} + {memory_usage ? Math.round(memory_usage / 1024 / 1024) + 'MB' : '--'}
({wm_memory_usage + ? Math.round(wm_memory_usage / 1024 / 1024) + 'MB' + : '--'}) {vcpus ? (vcpus / 100000).toFixed(1) + ' vCPUs' : '--'} - / {memory ? Math.round(memory / 1000000) + 'MB' : '--'} + / {memory ? Math.round(memory / 1024 / 1024) + 'MB' : '--'}