fix: remove useless metrics (#3962)

* fix: remove useless metrics

* fix: nit

* feat: update ee ref (renew key on start + vcpu usage)

---------

Co-authored-by: Ruben Fiszel <ruben@windmill.dev>

backend/.sqlx/query-021be0f26ea87e587e656b24a9a94538efbf54a1447a3898e19773789cfc9063.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "SELECT created_at FROM metrics WHERE id = 'author_count' ORDER BY created_at DESC LIMIT 1",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "created_at",
-        "type_info": "Timestamptz"
-      }
-    ],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "021be0f26ea87e587e656b24a9a94538efbf54a1447a3898e19773789cfc9063"
-}

backend/.sqlx/query-0cb84cbb9083d967cc8be1cccab5be61080c1003eef51eea41862b25c2b93de6.json

@@ -1,6 +1,6 @@
 {
   "db_name": "PostgreSQL",
-  "query": "INSERT INTO metrics (id, value) VALUES ('worker_usage', $1)",
+  "query": "INSERT INTO metrics (id, value) VALUES ('telemetry', $1)",
   "describe": {
     "columns": [],
     "parameters": {
@@ -10,5 +10,5 @@
     },
     "nullable": []
   },
-  "hash": "5ce9a9e0669b299b998bc9080d24f8d6f56eb049bec2e0745d27ca3a4fc0c296"
+  "hash": "0cb84cbb9083d967cc8be1cccab5be61080c1003eef51eea41862b25c2b93de6"
 }

backend/.sqlx/query-3667d72d23e8c35ab96d5e5d1fbfd94a8a9f74b6398e21f13c206c3f9427c6a9.json

@@ -1,12 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "DELETE FROM metrics \n                WHERE (id = 'author_count' OR id = 'operator_count' OR id = 'worker_usage') AND created_at < NOW() - INTERVAL '6 month'",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": []
-    },
-    "nullable": []
-  },
-  "hash": "3667d72d23e8c35ab96d5e5d1fbfd94a8a9f74b6398e21f13c206c3f9427c6a9"
-}

backend/.sqlx/query-8480792eefdd0f31a49afec0f91262a6973c068da5413b154eff9b9da90db95f.json

@@ -0,0 +1,44 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT worker, worker_instance, vcpus, ping_at, started_at FROM worker_ping WHERE ping_at > now() - interval '30 days'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "worker",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "worker_instance",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 2,
+        "name": "vcpus",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 3,
+        "name": "ping_at",
+        "type_info": "Timestamptz"
+      },
+      {
+        "ordinal": 4,
+        "name": "started_at",
+        "type_info": "Timestamptz"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      false,
+      true,
+      false,
+      false
+    ]
+  },
+  "hash": "8480792eefdd0f31a49afec0f91262a6973c068da5413b154eff9b9da90db95f"
+}

backend/.sqlx/query-c6b2791dd109c7bf40a40b557d9d5a140e70c49718fcb3a776b77f8f2ab901d4.json

@@ -1,15 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "INSERT INTO metrics (id, value) VALUES ('author_count', $1), ('operator_count', $2)",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Jsonb",
-        "Jsonb"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "c6b2791dd109c7bf40a40b557d9d5a140e70c49718fcb3a776b77f8f2ab901d4"
-}

backend/ee-repo-ref.txt

@@ -1 +1 @@
-9ef2d9c4aebfee15935634274a2c7c685331a865
+e9917f62ebc48bdd93f9ddda5a58ba6ce8a74ed9

backend/src/monitor.rs

@@ -8,7 +8,7 @@ use std::{
 };
 
 use rsmq_async::MultiplexedRsmq;
-use serde::{de::DeserializeOwned, Serialize};
+use serde::de::DeserializeOwned;
 use sqlx::{Pool, Postgres};
 use tokio::{
     join,
@@ -38,7 +38,6 @@ use windmill_common::{
     jobs::QueuedJob,
     oauth2::REQUIRE_PREEXISTING_USER_FOR_OAUTH,
     server::load_server_config,
-    stats_ee::get_user_usage,
     users::truncate_token,
     utils::{now_from_db, rd_string},
     worker::{
@@ -786,17 +785,10 @@ pub async fn monitor_db(
         }
     };
 
-    let save_usage_metrics_f = async {
-        if !initial_load && server_mode {
-            save_usage_metrics(&db).await;
-        }
-    };
-
     join!(
         expired_items_f,
         zombie_jobs_f,
         expose_queue_metrics_f,
-        save_usage_metrics_f,
         verify_license_key_f
     );
 }
@@ -871,89 +863,6 @@ pub async fn expose_queue_metrics(db: &Pool<Postgres>) {
     }
 }
 
-#[derive(Serialize)]
-struct WorkerUsage {
-    worker: String,
-    worker_instance: String,
-    vcpus: Option<i64>,
-    memory: Option<i64>,
-}
-
-pub async fn save_usage_metrics(db: &Pool<Postgres>) {
-    let tx = db.begin().await;
-
-    if let Ok(mut tx) = tx {
-        let last_check = sqlx::query_scalar!(
-            "SELECT created_at FROM metrics WHERE id = 'author_count' ORDER BY created_at DESC LIMIT 1"
-        )
-        .fetch_optional(db)
-        .await
-        .unwrap_or(Some(chrono::Utc::now()));
-
-        let random_nb = rand::random::<i64>();
-
-        // save author and operator count every ~24 hours
-        if last_check
-            .map(|last_check| {
-                chrono::Utc::now() - last_check
-                    > chrono::Duration::hours(24) - chrono::Duration::minutes(random_nb % 60)
-            })
-            .unwrap_or(true)
-        {
-            let user_usage = get_user_usage(&mut *tx).await.ok();
-
-            if let Some(user_usage) = user_usage {
-                sqlx::query!(
-                    "INSERT INTO metrics (id, value) VALUES ('author_count', $1), ('operator_count', $2)",
-                    serde_json::json!(user_usage.author_count.unwrap_or(0)),
-                    serde_json::json!(user_usage.operator_count.unwrap_or(0))
-                )
-                .execute(&mut *tx)
-                .await
-                .ok();
-            }
-
-            // clean metrics older than 6 months (including worker usage)
-            sqlx::query!(
-                "DELETE FROM metrics 
-                WHERE (id = 'author_count' OR id = 'operator_count' OR id = 'worker_usage') AND created_at < NOW() - INTERVAL '6 month'"
-            )
-            .execute(&mut *tx)
-            .await
-            .ok();
-        }
-
-        // save worker usage every ~60 minutes
-        if last_check
-            .map(|last_check| {
-                chrono::Utc::now() - last_check
-                    > chrono::Duration::minutes(60) - chrono::Duration::seconds(random_nb % 300)
-            })
-            .unwrap_or(true)
-        {
-            let worker_usage = sqlx::query_as!(
-                WorkerUsage,
-                "SELECT worker, worker_instance, vcpus, memory FROM worker_ping WHERE ping_at > NOW() - INTERVAL '2 minutes'"
-            )
-            .fetch_all(&mut *tx)
-            .await
-            .ok();
-
-            if let Some(worker_usage) = worker_usage {
-                sqlx::query!(
-                    "INSERT INTO metrics (id, value) VALUES ('worker_usage', $1)",
-                    serde_json::json!(worker_usage)
-                )
-                .execute(&mut *tx)
-                .await
-                .ok();
-            }
-        }
-
-        tx.commit().await.ok();
-    }
-}
-
 pub async fn reload_server_config(db: &Pool<Postgres>) {
     let config = load_server_config(&db).await;
     if let Err(e) = config {