starred/windmill
1
0
Fork 0
mirror of https://github.com/windmill-labs/windmill.git synced 2026-10-09 08:02:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: collect vcpus and accounts usage (#2635)

Browse Source
This commit is contained in:
HugoCasa
2023-11-16 13:52:56 +01:00
committed by GitHub
parent 1de15a800c
commit 93851ffd37
7 changed files with 90 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/.sqlx/query-1c67cce8cdf6e56752643d6382ebaab76e24581009ca5e8d8b5ce3f199b689ab.json → backend/.sqlx/query-cdfc8e9af4414b7ee7525bc74d9caad74b5479c64337c401a7a9550a3434cdfa.json
Generated
+2 -2
View File
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) FROM worker_ping WHERE ping_at > NOW() - INTERVAL '5 minutes'",
"query": "SELECT COUNT(*) FROM worker_ping WHERE ping_at > NOW() - INTERVAL '2 minutes'",
"describe": {
"columns": [
{
@@ -16,5 +16,5 @@
null
]
},
"hash": "1c67cce8cdf6e56752643d6382ebaab76e24581009ca5e8d8b5ce3f199b689ab"
"hash": "cdfc8e9af4414b7ee7525bc74d9caad74b5479c64337c401a7a9550a3434cdfa"
}
backend/.sqlx/query-e70835de2b70a01a2e85f44f3cb80f41288d8e68b762de80ae724ed69ef1908c.json
Generated
+26
View File
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT author.count as author_count, operator.count as operator_count FROM (SELECT count(*)::INT FROM usr where usr.operator IS false) as author, (SELECT count(*)::INT FROM usr where usr.operator IS true) as operator",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "author_count",
"type_info": "Int4"
},
{
"ordinal": 1,
"name": "operator_count",
"type_info": "Int4"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "e70835de2b70a01a2e85f44f3cb80f41288d8e68b762de80ae724ed69ef1908c"
}
backend/src/main.rs
+3 -14
View File
@@ -9,7 +9,6 @@
use gethostname::gethostname;
use git_version::git_version;
use rand::Rng;
use serde::Deserialize;
use sqlx::{postgres::PgListener, Pool, Postgres};
use std::{
net::{IpAddr, Ipv4Addr, SocketAddr},
@@ -30,7 +29,7 @@ use windmill_common::{
RETENTION_PERIOD_SECS_SETTING,
},
stats::schedule_stats,
utils::rd_string,
utils::{rd_string, Mode},
worker::{reload_custom_tags_setting, WORKER_GROUP},
DB, METRICS_ADDR, METRICS_ENABLED,
};
@@ -59,14 +58,6 @@ mod monitor;
#[cfg(feature = "pg_embed")]
mod pg_embed;
#[derive(Deserialize, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum Mode {
Worker,
Server,
Standalone,
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
dotenv::dotenv().ok();
@@ -460,10 +451,8 @@ Windmill Community Edition {GIT_VERSION}
Ok(()) as anyhow::Result<()>
};
if mode == Mode::Server || mode == Mode::Standalone {
let instance_name = rd_string(8);
schedule_stats(&db, instance_name, &HTTP_CLIENT).await;
}
let instance_name = rd_string(8);
schedule_stats(instance_name, mode, &db, &HTTP_CLIENT).await;
futures::try_join!(shutdown_signal, server_f, metrics_f, workers_f, monitor_f)?;
} else {
backend/windmill-api/src/settings.rs
+7 -1
View File
@@ -172,7 +172,13 @@ pub async fn get_global_setting(
pub async fn send_stats(Extension(db): Extension<DB>, authed: ApiAuthed) -> Result<String> {
require_super_admin(&db, &authed.email).await?;
windmill_common::stats::send_stats(&"manual".to_string(), &HTTP_CLIENT, &db).await?;
windmill_common::stats::send_stats(
&"manual".to_string(),
&windmill_common::utils::Mode::Server,
&HTTP_CLIENT,
&db,
)
.await?;
Ok("Sent stats".to_string())
}
backend/windmill-common/src/stats.rs
+39 -4
View File
@@ -4,7 +4,7 @@ use crate::{
error::{to_anyhow, Result},
global_settings::DISABLE_STATS_SETTING,
scripts::ScriptLang,
utils::{get_uid, GIT_VERSION},
utils::{get_uid, Mode, GIT_VERSION},
DB,
};
use chrono::Utc;
@@ -34,15 +34,22 @@ pub async fn get_disable_stats_setting(db: &DB) -> bool {
false
}
pub async fn schedule_stats(db: &DB, instance_name: String, http_client: &reqwest::Client) -> () {
pub async fn schedule_stats(
instance_name: String,
mode: Mode,
db: &DB,
http_client: &reqwest::Client,
) -> () {
let http_client = http_client.clone();
let db = db.clone();
tokio::spawn(async move {
tokio::time::sleep(tokio::time::Duration::from_secs(180)).await;
loop {
let disabled = get_disable_stats_setting(&db).await;
if !disabled {
tracing::info!("Sending stats");
let result = send_stats(&instance_name, &http_client, &db).await;
let result = send_stats(&instance_name, &mode, &http_client, &db).await;
if result.is_err() {
tracing::info!("Error sending stats: {}", result.err().unwrap());
} else {
@@ -83,6 +90,7 @@ struct JobsUsage {
pub async fn send_stats(
instance_name: &String,
mode: &Mode,
http_client: &reqwest::Client,
db: &DB,
) -> Result<()> {
@@ -108,13 +116,32 @@ pub async fn send_stats(
.collect::<Vec<serde_json::Value>>();
let workers_usage = sqlx::query!(
"SELECT COUNT(*) FROM worker_ping WHERE ping_at > NOW() - INTERVAL '5 minutes'"
"SELECT COUNT(*) FROM worker_ping WHERE ping_at > NOW() - INTERVAL '2 minutes'"
)
.fetch_one(db)
.await?
.count
.unwrap_or(0);
let users_count = sqlx::query!("SELECT author.count as author_count, operator.count as operator_count FROM (SELECT count(*)::INT FROM usr where usr.operator IS false) as author, (SELECT count(*)::INT FROM usr where usr.operator IS true) as operator")
.fetch_one(db)
.await?;
let vcpus = std::process::Command::new("cat")
.args(["/sys/fs/cgroup/cpu.max"])
.output()
.ok()
.map(|o| {
String::from_utf8_lossy(&o.stdout)
.to_string()
.split(" ")
.map(|s| s.to_string())
.collect::<Vec<String>>()
.get(0)
.map(|s| s.to_string())
})
.flatten();
let payload = serde_json::json!({
"uid": uid,
"version": GIT_VERSION,
@@ -122,8 +149,16 @@ pub async fn send_stats(
"jobs_usage": jobs_usage,
"login_type_usage": login_type_usage,
"workers_usage": workers_usage,
"users_usage": {
"author_count": users_count.author_count.unwrap_or(0),
"operator_count": users_count.operator_count.unwrap_or(0),
},
"mode": mode,
"vcpus": vcpus,
});
tracing::info!("Sending stats: {:#?}", payload);
let request = http_client
.post("https://hub.windmill.dev/stats")
.body(serde_json::to_string(&payload).map_err(to_anyhow)?)
backend/windmill-common/src/utils.rs
+8
View File
@@ -162,3 +162,11 @@ pub async fn get_uid(db: &DB) -> Result<String> {
Ok(uid)
}
#[derive(Serialize, Deserialize, PartialEq, Clone)]
#[serde(rename_all = "lowercase")]
pub enum Mode {
Worker,
Server,
Standalone,
}
frontend/src/lib/components/InstanceSettings.svelte
+5 -3
View File
@@ -18,7 +18,7 @@
import CustomOauth from './CustomOauth.svelte'
import { AlertTriangle, Plus } from 'lucide-svelte'
import CustomSso from './CustomSso.svelte'
import AuthentikSetting from "$lib/components/AuthentikSetting.svelte";
import AuthentikSetting from '$lib/components/AuthentikSetting.svelte'
export let tab: string = 'Core'
export let hideTabs: boolean = false
@@ -177,8 +177,10 @@
<ul class="list-disc list-inside pl-2">
<li>version</li>
<li>number and total duration of jobs</li>
<li>accounts usage</li>
<li>login type usage</li>
<li>workers usage</li>
<li>vcpus usage</li>
</ul>
</div>
{#if $enterpriseLicense}
@@ -216,7 +218,7 @@
<KeycloakSetting bind:value={oauths['keycloak']} />
<AuthentikSetting bind:value={oauths['authentik']} />
{#each Object.keys(oauths) as k}
{#if !['authentik','google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'keycloak', 'slack'].includes(k) && 'login_config' in oauths[k]}
{#if !['authentik', 'google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'keycloak', 'slack'].includes(k) && 'login_config' in oauths[k]}
{#if oauths[k]}
<div class="flex flex-col gap-2 pb-4">
<div class="flex flex-row items-center gap-2">
@@ -281,7 +283,7 @@
<div class="py-1" />
{#each Object.keys(oauths) as k}
{#if !['authentik','google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'keycloak', 'slack'].includes(k) && !('login_config' in oauths[k])}
{#if !['authentik', 'google', 'microsoft', 'github', 'gitlab', 'jumpcloud', 'okta', 'keycloak', 'slack'].includes(k) && !('login_config' in oauths[k])}
{#if oauths[k]}
<div class="flex flex-col gap-2 pb-4">
<div class="flex flex-row items-center gap-2">