feat: add new stats (#2568)

* feat: add new stats

* feat: use license key id + manually send usage

backend/.sqlx/query-08e4a2dc49c75aa356f3cc75a4abd8fc61409776d641ddb592a4c731e61a0468.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT login_type, COUNT(*) FROM password GROUP BY login_type",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "login_type",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 1,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      false,
+      null
+    ]
+  },
+  "hash": "08e4a2dc49c75aa356f3cc75a4abd8fc61409776d641ddb592a4c731e61a0468"
+}

backend/.sqlx/query-1c67cce8cdf6e56752643d6382ebaab76e24581009ca5e8d8b5ce3f199b689ab.json

@@ -0,0 +1,20 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT COUNT(*) FROM worker_ping WHERE ping_at > NOW() - INTERVAL '5 minutes'",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "count",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": []
+    },
+    "nullable": [
+      null
+    ]
+  },
+  "hash": "1c67cce8cdf6e56752643d6382ebaab76e24581009ca5e8d8b5ce3f199b689ab"
+}

backend/src/ee.rs

@@ -2,7 +2,8 @@
 use windmill_common::error;
 
 pub async fn set_license_key(license_key: String) -> anyhow::Result<()> {
-    use windmill_api::{ee::validate_license_key, LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID};
+    use windmill_api::ee::validate_license_key;
+    use windmill_common::ee::{LICENSE_KEY, LICENSE_KEY_ID, LICENSE_KEY_VALID};
 
     let id = validate_license_key(license_key.clone()).await?;
     {
@@ -24,7 +25,7 @@ pub async fn set_license_key(license_key: String) -> anyhow::Result<()> {
 
 #[cfg(feature = "enterprise")]
 pub async fn verify_license_key() -> error::Result<()> {
-    use windmill_api::{LICENSE_KEY, LICENSE_KEY_VALID};
+    use windmill_common::ee::{LICENSE_KEY, LICENSE_KEY_VALID};
     use windmill_common::error::to_anyhow;
 
     let expiry_nb = LICENSE_KEY

backend/src/monitor.rs

@@ -41,7 +41,7 @@ use windmill_worker::{
 use crate::ee::verify_license_key;
 
 #[cfg(feature = "enterprise")]
-use windmill_api::LICENSE_KEY_VALID;
+use windmill_common::ee::LICENSE_KEY_VALID;
 
 use crate::ee::set_license_key;
 

backend/windmill-api/openapi.yaml

@@ -650,6 +650,20 @@ paths:
               schema:
                 type: string
 
+  /settings/send_stats:
+    post:
+      summary: send stats
+      operationId: sendStats
+      tags:
+        - setting
+      responses:
+        "200":
+          description: status
+          content:
+            text/plain::
+              schema:
+                type: string
+
   /users/email:
     get:
       summary: get current user email (if logged in)

backend/windmill-api/src/jobs.rs

@@ -1569,7 +1569,7 @@ async fn check_tag_available_for_workspace(w_id: &str, tag: &Option<String>) ->
 
 #[cfg(feature = "enterprise")]
 pub async fn check_license_key_valid() -> error::Result<()> {
-    use crate::LICENSE_KEY_VALID;
+    use windmill_common::ee::LICENSE_KEY_VALID;
 
     let valid = *LICENSE_KEY_VALID.read().await;
     if !valid {

backend/windmill-api/src/lib.rs

@@ -104,10 +104,6 @@ lazy_static::lazy_static! {
         connects: HashMap::new(),
         slack: None
     }));
-
-    pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
-    pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
-    pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
 }
 
 pub async fn run_server(
@@ -337,6 +333,8 @@ async fn ee_license() -> &'static str {
 
 #[cfg(feature = "enterprise")]
 async fn ee_license() -> String {
+    use windmill_common::ee::LICENSE_KEY_ID;
+
     LICENSE_KEY_ID.read().await.clone()
 }
 

backend/windmill-api/src/settings.rs

@@ -12,6 +12,7 @@ use crate::{
     db::{ApiAuthed, DB},
     ee::validate_license_key,
     utils::require_super_admin,
+    HTTP_CLIENT,
 };
 
 use axum::{
@@ -24,7 +25,7 @@ use mail_send::{mail_builder::MessageBuilder, SmtpClientBuilder};
 use serde::Deserialize;
 use tokio::time::timeout;
 use windmill_common::{
-    error::{self, to_anyhow, JsonResult},
+    error::{self, to_anyhow, JsonResult, Result},
     global_settings::ENV_SETTINGS,
     server::Smtp,
 };
@@ -38,6 +39,7 @@ pub fn global_service() -> Router {
         )
         .route("/test_smtp", post(test_email))
         .route("/test_license_key", post(test_license_key))
+        .route("/send_stats", post(send_stats))
 }
 
 #[derive(Deserialize)]
@@ -158,3 +160,10 @@ pub async fn get_global_setting(
 
     Ok(Json(value.unwrap_or_else(|| serde_json::Value::Null)))
 }
+
+pub async fn send_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Result<String> {
+    require_super_admin(&db, &authed.email).await?;
+    windmill_common::stats::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?;
+
+    Ok("Sent stats".to_string())
+}

backend/windmill-common/src/ee.rs

@@ -0,0 +1,8 @@
+use std::sync::Arc;
+use tokio::sync::RwLock;
+
+lazy_static::lazy_static! {
+  pub static ref LICENSE_KEY_VALID: Arc<RwLock<bool>> = Arc::new(RwLock::new(true));
+  pub static ref LICENSE_KEY_ID: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+  pub static ref LICENSE_KEY: Arc<RwLock<String>> = Arc::new(RwLock::new("".to_string()));
+}

backend/windmill-common/src/lib.rs

@@ -17,6 +17,7 @@ use sqlx::{Pool, Postgres};
 
 pub mod apps;
 pub mod db;
+pub mod ee;
 pub mod error;
 pub mod external_ip;
 pub mod flow_status;

backend/windmill-common/src/stats.rs

@@ -2,11 +2,11 @@ use std::str::FromStr;
 
 use crate::{
     error::{to_anyhow, Result},
-    global_settings::{DISABLE_STATS_SETTING, UNIQUE_ID_SETTING},
-    utils::GIT_VERSION,
+    global_settings::DISABLE_STATS_SETTING,
+    scripts::ScriptLang,
+    utils::{get_uid, GIT_VERSION},
     DB,
 };
-
 use chrono::Utc;
 use cron::Schedule;
 
@@ -74,24 +74,54 @@ pub async fn schedule_stats(db: &DB, instance_name: String, http_client: &reqwes
     });
 }
 
+#[derive(Debug, sqlx::FromRow, serde::Serialize)]
+struct JobsUsage {
+    language: Option<ScriptLang>,
+    total_duration: i64,
+    count: i64,
+}
+
 pub async fn send_stats(
     instance_name: &String,
     http_client: &reqwest::Client,
     db: &DB,
 ) -> Result<()> {
-    let uid = sqlx::query_scalar!(
-        "SELECT value FROM global_settings WHERE name = $1",
-        UNIQUE_ID_SETTING
+    let uid = get_uid(db).await?;
+
+    let jobs_usage = sqlx::query_as::<_, JobsUsage>(
+        "SELECT language, COUNT(*) as count, SUM(duration_ms)::BIGINT as total_duration FROM completed_job GROUP BY language",
     )
-    .fetch_one(db)
+    .fetch_all(db)
     .await?;
 
-    let uid = serde_json::from_value::<String>(uid).map_err(to_anyhow)?;
+    let login_type_usage =
+        sqlx::query!("SELECT login_type, COUNT(*) FROM password GROUP BY login_type")
+            .fetch_all(db)
+            .await?
+            .into_iter()
+            .map(|r| {
+                serde_json::json!({
+                    "login_type": r.login_type,
+                    "count": r.count.unwrap_or(0),
+                })
+            })
+            .collect::<Vec<serde_json::Value>>();
+
+    let workers_usage = sqlx::query!(
+        "SELECT COUNT(*) FROM worker_ping WHERE ping_at > NOW() - INTERVAL '5 minutes'"
+    )
+    .fetch_one(db)
+    .await?
+    .count
+    .unwrap_or(0);
 
     let payload = serde_json::json!({
         "uid": uid,
         "version": GIT_VERSION,
         "instance_name": instance_name,
+        "jobs_usage": jobs_usage,
+        "login_type_usage": login_type_usage,
+        "workers_usage": workers_usage,
     });
 
     let request = http_client

backend/windmill-common/src/utils.rs

@@ -6,6 +6,7 @@
  * LICENSE-AGPL for a copy of the license.
  */
 
+use crate::ee::LICENSE_KEY_ID;
 use crate::error::{to_anyhow, Error, Result};
 use crate::global_settings::UNIQUE_ID_SETTING;
 use crate::DB;
@@ -103,13 +104,7 @@ pub async fn http_get_from_hub(
     query_params: Option<Vec<(&str, String)>>,
     db: &Pool<Postgres>,
 ) -> Result<reqwest::Response> {
-    let uid = sqlx::query_scalar!(
-        "SELECT value FROM global_settings WHERE name = $1",
-        UNIQUE_ID_SETTING
-    )
-    .fetch_optional(db)
-    .await?
-    .map(|v| serde_json::from_value::<String>(v));
+    let uid = get_uid(db).await;
 
     let mut request = http_client.get(url).header(
         "Accept",
@@ -120,12 +115,10 @@ pub async fn http_get_from_hub(
         },
     );
 
-    if let Some(uid) = uid {
-        if let Ok(uid) = uid {
-            request = request.header("X-uid", uid);
-        } else {
-            tracing::info!("Invalid uid in global settings: {}", uid.err().unwrap())
-        }
+    if let Ok(uid) = uid {
+        request = request.header("X-uid", uid);
+    } else {
+        tracing::info!("No valid uid found: {}", uid.err().unwrap())
     }
 
     if let Some(query_params) = query_params {
@@ -152,3 +145,20 @@ pub fn calculate_hash(s: &str) -> String {
     hasher.update(s);
     format!("{:x}", hasher.finalize())
 }
+
+pub async fn get_uid(db: &DB) -> Result<String> {
+    let mut uid = LICENSE_KEY_ID.read().await.clone();
+
+    if uid == "" {
+        let uid_value = sqlx::query_scalar!(
+            "SELECT value FROM global_settings WHERE name = $1",
+            UNIQUE_ID_SETTING
+        )
+        .fetch_one(db)
+        .await?;
+
+        uid = serde_json::from_value::<String>(uid_value).map_err(to_anyhow)?;
+    }
+
+    Ok(uid)
+}

frontend/src/lib/components/InstanceSettings.svelte

@@ -143,6 +143,11 @@
 	]
 
 	let oauth_name = 'custom'
+
+	async function sendStats() {
+		await SettingService.sendStats()
+		sendUserToast('Usage sent')
+	}
 </script>
 
 <div class="pb-8">
@@ -164,8 +169,23 @@
 						<div class="text-secondary pb-4 text-xs">
 							Anonymous usage data is collected to help improve Windmill.
 							<br />The following information is collected:
-							<ul class="list-disc list-inside pl-2"><li>version</li></ul>
+							<ul class="list-disc list-inside pl-2">
+								<li>version</li>
+								<li>number and total duration of jobs</li>
+								<li>login type usage</li>
+								<li>workers usage</li>
+							</ul>
 						</div>
+						{#if $enterpriseLicense}
+							<Button
+								on:click={sendStats}
+								variant="border"
+								color="light"
+								btnClasses="w-auto"
+								wrapperClasses="mb-4"
+								size="xs">Send usage</Button
+							>
+						{/if}
 					{/if}
 					{#if category == 'SSO/OAuth'}
 						<div>