From 79948d09bb25ec73158946c0e64f659746ae47b0 Mon Sep 17 00:00:00 2001
From: HugoCasa <hugo@casademont.ch>
Date: Wed, 29 May 2024 18:16:41 +0200
Subject: [PATCH] fix: worker memory usage (#3845)

---
 backend/Cargo.lock                            |   2 -
 backend/Cargo.toml                            |   2 -
 backend/windmill-common/Cargo.toml            |   1 -
 backend/windmill-common/src/worker.rs         | 117 ++++++++++++++++--
 .../(root)/(logged)/workers/+page.svelte      |   2 +-
 5 files changed, 107 insertions(+), 17 deletions(-)

diff --git a/backend/Cargo.lock b/backend/Cargo.lock
index a27bb873a4..1aad14d250 100644
--- a/backend/Cargo.lock
+++ b/backend/Cargo.lock
@@ -8313,7 +8313,6 @@ dependencies = [
  "libc",
  "ntapi",
  "once_cell",
- "rayon",
  "windows",
 ]
 
@@ -9873,7 +9872,6 @@ dependencies = [
  "serde_json",
  "sha2 0.10.8",
  "sqlx",
- "sysinfo",
  "thiserror",
  "tikv-jemalloc-ctl",
  "tokio",
diff --git a/backend/Cargo.toml b/backend/Cargo.toml
index 7d0b86d45f..ae284e1db0 100644
--- a/backend/Cargo.toml
+++ b/backend/Cargo.toml
@@ -249,8 +249,6 @@ tar = "^0"
 http = "^1"
 async-stream = "^0"
 
-sysinfo = "0.30.12"
-
 tikv-jemallocator = { version = "0.5" }
 tikv-jemalloc-sys  = { version = "^0.5" }
 tikv-jemalloc-ctl = { version = "^0.5" }
diff --git a/backend/windmill-common/Cargo.toml b/backend/windmill-common/Cargo.toml
index deab5c347e..634b3ae4e3 100644
--- a/backend/windmill-common/Cargo.toml
+++ b/backend/windmill-common/Cargo.toml
@@ -50,7 +50,6 @@ aws-sdk-sts = { workspace = true, optional = true }
 indexmap.workspace = true
 bytes = { workspace = true, optional = true }
 mail-send.workspace = true
-sysinfo.workspace = true
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 tikv-jemalloc-ctl = { optional = true, workspace = true }
diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs
index 52c2129929..1b42d683fc 100644
--- a/backend/windmill-common/src/worker.rs
+++ b/backend/windmill-common/src/worker.rs
@@ -7,7 +7,6 @@ use std::{
     collections::{HashMap, HashSet},
     sync::{atomic::AtomicBool, Arc},
 };
-use sysinfo::{MemoryRefreshKind, System};
 use tokio::sync::RwLock;
 
 use crate::{error, global_settings::CUSTOM_TAGS_SETTING, server::ServerConfig, DB};
@@ -169,21 +168,117 @@ pub fn get_vcpus() -> Option<i64> {
 }
 
 pub fn get_memory() -> Option<i64> {
-    let mut sys = System::new();
-    sys.refresh_memory();
-    let limits = sys.cgroup_limits();
+    let mut memory = std::process::Command::new("cat")
+        .args(["/sys/fs/cgroup/memory.max"])
+        .output()
+        .ok()
+        .map(|o| {
+            String::from_utf8_lossy(&o.stdout)
+                .to_string()
+                .trim()
+                .parse::<i64>()
+                .ok()
+        })
+        .flatten();
 
-    if let Some(limits) = limits {
-        i64::try_from(limits.total_memory).ok()
-    } else {
-        None
+    if memory.is_none() {
+        memory = std::process::Command::new("cat")
+            .args(["/sys/fs/cgroup/memory/memory.limit_in_bytes"])
+            .output()
+            .ok()
+            .map(|o| {
+                String::from_utf8_lossy(&o.stdout)
+                    .to_string()
+                    .trim()
+                    .parse::<i64>()
+                    .ok()
+            })
+            .flatten()
     }
+
+    memory
 }
 
 pub fn get_worker_memory_usage() -> Option<i64> {
-    let mut sys = System::new();
-    sys.refresh_memory_specifics(MemoryRefreshKind::new().with_ram());
-    i64::try_from(sys.used_memory()).ok()
+    let mut total_memory_usage = std::process::Command::new("cat")
+        .args(["/sys/fs/cgroup/memory.current"])
+        .output()
+        .ok()
+        .map(|o| {
+            String::from_utf8_lossy(&o.stdout)
+                .to_string()
+                .trim()
+                .parse::<i64>()
+                .ok()
+        })
+        .flatten();
+
+    if total_memory_usage.is_none() {
+        total_memory_usage = std::process::Command::new("cat")
+            .args(["/sys/fs/cgroup/memory/memory.usage_in_bytes"])
+            .output()
+            .ok()
+            .map(|o| {
+                String::from_utf8_lossy(&o.stdout)
+                    .to_string()
+                    .trim()
+                    .parse::<i64>()
+                    .ok()
+            })
+            .flatten()
+    }
+
+    match total_memory_usage {
+        Some(total_memory_usage) => {
+            let mut inactive_file = std::process::Command::new("cat")
+                .args(["/sys/fs/cgroup/memory.stat"])
+                .output()
+                .ok()
+                .map(|o| {
+                    String::from_utf8_lossy(&o.stdout)
+                        .to_string()
+                        .split("\n")
+                        .find(|s| s.starts_with("inactive_file"))
+                        .map(|s| {
+                            s.split(" ")
+                                .collect::<Vec<&str>>()
+                                .get(1)
+                                .map(|s| s.trim().parse::<i64>().ok())
+                                .flatten()
+                        })
+                        .flatten()
+                })
+                .flatten();
+
+            if inactive_file.is_none() {
+                inactive_file = std::process::Command::new("cat")
+                    .args(["/sys/fs/cgroup/memory/memory.stat"])
+                    .output()
+                    .ok()
+                    .map(|o| {
+                        String::from_utf8_lossy(&o.stdout)
+                            .to_string()
+                            .split("\n")
+                            .find(|s| s.starts_with("total_inactive_file"))
+                            .map(|s| {
+                                s.split(" ")
+                                    .collect::<Vec<&str>>()
+                                    .get(1)
+                                    .map(|s| s.trim().parse::<i64>().ok())
+                                    .flatten()
+                            })
+                            .flatten()
+                    })
+                    .flatten();
+            }
+
+            match inactive_file {
+                Some(inactive_file) => Some(total_memory_usage - inactive_file),
+                None => None,
+            }
+        }
+        None => None,
+    }
 }
 
 pub fn get_windmill_memory_usage() -> Option<i64> {
diff --git a/frontend/src/routes/(root)/(logged)/workers/+page.svelte b/frontend/src/routes/(root)/(logged)/workers/+page.svelte
index 4e4b01b230..2865957002 100644
--- a/frontend/src/routes/(root)/(logged)/workers/+page.svelte
+++ b/frontend/src/routes/(root)/(logged)/workers/+page.svelte
@@ -346,7 +346,7 @@
 							<Cell head>Current job</Cell>
 							<Cell head>Occupancy rate</Cell>
 						{/if}
-						<Cell head>Memory usage<br />(Windmill usage)</Cell>
+						<Cell head>Memory usage<br />(Windmill)</Cell>
 						<Cell head>Limits</Cell>
 						<Cell head>Version</Cell>
 						<Cell head last>Liveness</Cell>