diff --git a/backend/.sqlx/query-143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d.json b/backend/.sqlx/query-143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d.json new file mode 100644 index 0000000000..3514370ec8 --- /dev/null +++ b/backend/.sqlx/query-143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') as \"exists!\"", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists!", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d" +} diff --git a/backend/.sqlx/query-1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6.json b/backend/.sqlx/query-1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6.json new file mode 100644 index 0000000000..987acc6d1b --- /dev/null +++ b/backend/.sqlx/query-1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n schemaname || '.' || relname as \"table_name!\",\n pg_total_relation_size(relid) as \"total_size_bytes!\",\n pg_size_pretty(pg_total_relation_size(relid)) as \"total_size_pretty!\"\n FROM pg_catalog.pg_statio_user_tables\n ORDER BY pg_total_relation_size(relid) DESC\n LIMIT 15", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "table_name!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "total_size_bytes!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "total_size_pretty!", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null + ] + }, + "hash": "1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6" +} diff --git a/backend/.sqlx/query-26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2.json b/backend/.sqlx/query-26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2.json new file mode 100644 index 0000000000..2ab912644a --- /dev/null +++ b/backend/.sqlx/query-26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT value FROM global_settings WHERE name = 'retention_period_secs'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "value", + "type_info": "Jsonb" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false + ] + }, + "hash": "26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2" +} diff --git a/backend/.sqlx/query-2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720.json b/backend/.sqlx/query-2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720.json new file mode 100644 index 0000000000..007c6fdcb2 --- /dev/null +++ b/backend/.sqlx/query-2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720.json @@ -0,0 +1,32 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n ws.workspace_id as \"workspace_id!\",\n dt.key as \"name!\",\n dt.value->>'table_name' as \"table_name\"\n FROM workspace_settings ws,\n jsonb_each(ws.datatable) dt\n WHERE dt.value->>'resource_type' = 'instance'\n AND dt.value->>'table_name' IS NOT NULL", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 1, + "name": "name!", + "type_info": "Text" + }, + { + "ordinal": 2, + "name": "table_name", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + null, + null + ] + }, + "hash": "2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720" +} diff --git a/backend/.sqlx/query-384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0.json b/backend/.sqlx/query-384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0.json new file mode 100644 index 0000000000..cb033816d1 --- /dev/null +++ b/backend/.sqlx/query-384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT pg_database_size(current_database()) as size_bytes, pg_size_pretty(pg_database_size(current_database())) as size_pretty", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "size_bytes", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "size_pretty", + "type_info": "Text" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0" +} diff --git a/backend/.sqlx/query-7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84.json b/backend/.sqlx/query-7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84.json new file mode 100644 index 0000000000..304027071a --- /dev/null +++ b/backend/.sqlx/query-7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84.json @@ -0,0 +1,40 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n c.relname as \"table_name!\",\n pg_total_relation_size(c.oid) as \"size_bytes!\",\n pg_size_pretty(pg_total_relation_size(c.oid)) as \"size_pretty!\",\n COALESCE(c.reltuples, 0) as \"estimated_rows!\"\n FROM pg_class c\n JOIN pg_namespace n ON n.oid = c.relnamespace\n WHERE n.nspname = 'public' AND c.relname = ANY($1)", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "table_name!", + "type_info": "Name" + }, + { + "ordinal": 1, + "name": "size_bytes!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "size_pretty!", + "type_info": "Text" + }, + { + "ordinal": 3, + "name": "estimated_rows!", + "type_info": "Float4" + } + ], + "parameters": { + "Left": [ + "NameArray" + ] + }, + "nullable": [ + false, + null, + null, + null + ] + }, + "hash": "7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84" +} diff --git a/backend/.sqlx/query-87d07998fe8373f5b89eccf6f0528c02e389bf827d935d867430ad3459104dd9.json b/backend/.sqlx/query-87d07998fe8373f5b89eccf6f0528c02e389bf827d935d867430ad3459104dd9.json new file mode 100644 index 0000000000..24d15601e7 --- /dev/null +++ b/backend/.sqlx/query-87d07998fe8373f5b89eccf6f0528c02e389bf827d935d867430ad3459104dd9.json @@ -0,0 +1,20 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT COUNT(*) as cnt FROM pg_stat_activity WHERE state = 'active'", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "cnt", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null + ] + }, + "hash": "87d07998fe8373f5b89eccf6f0528c02e389bf827d935d867430ad3459104dd9" +} diff --git a/backend/.sqlx/query-a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00.json b/backend/.sqlx/query-a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00.json new file mode 100644 index 0000000000..cb6ed918c4 --- /dev/null +++ b/backend/.sqlx/query-a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00.json @@ -0,0 +1,22 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT AVG(pg_column_size(result))::bigint as \"avg_size\"\n FROM (\n SELECT result FROM v2_job_completed\n WHERE completed_at > now() - interval '30 days'\n AND result IS NOT NULL\n ORDER BY completed_at DESC\n LIMIT $1\n ) sub", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "avg_size", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + null + ] + }, + "hash": "a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00" +} diff --git a/backend/.sqlx/query-b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30.json b/backend/.sqlx/query-b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30.json new file mode 100644 index 0000000000..a01296f654 --- /dev/null +++ b/backend/.sqlx/query-b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30.json @@ -0,0 +1,26 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT MIN(completed_at) as oldest, COUNT(*) as total FROM v2_job_completed", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "oldest", + "type_info": "Timestamptz" + }, + { + "ordinal": 1, + "name": "total", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null + ] + }, + "hash": "b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30" +} diff --git a/backend/.sqlx/query-bc54ea311580a0525c1f36aaa543c5798e6f7aca1e6e564330766d77038ef0e3.json b/backend/.sqlx/query-bc54ea311580a0525c1f36aaa543c5798e6f7aca1e6e564330766d77038ef0e3.json new file mode 100644 index 0000000000..c68971b7ba --- /dev/null +++ b/backend/.sqlx/query-bc54ea311580a0525c1f36aaa543c5798e6f7aca1e6e564330766d77038ef0e3.json @@ -0,0 +1,44 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n schemaname || '.' || relname as \"table_name!\",\n COALESCE(n_live_tup, 0) as \"live_tuples!\",\n COALESCE(n_dead_tup, 0) as \"dead_tuples!\",\n last_autovacuum as \"last_autovacuum\",\n last_autoanalyze as \"last_autoanalyze\"\n FROM pg_stat_user_tables\n ORDER BY n_dead_tup DESC\n LIMIT 15", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "table_name!", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "live_tuples!", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "dead_tuples!", + "type_info": "Int8" + }, + { + "ordinal": 3, + "name": "last_autovacuum", + "type_info": "Timestamptz" + }, + { + "ordinal": 4, + "name": "last_autoanalyze", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + null, + null, + null, + true, + true + ] + }, + "hash": "bc54ea311580a0525c1f36aaa543c5798e6f7aca1e6e564330766d77038ef0e3" +} diff --git a/backend/.sqlx/query-dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7.json b/backend/.sqlx/query-dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7.json new file mode 100644 index 0000000000..33eb1fbf7e --- /dev/null +++ b/backend/.sqlx/query-dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7.json @@ -0,0 +1,46 @@ +{ + "db_name": "PostgreSQL", + "query": "SELECT\n c.id as \"id!\",\n c.workspace_id as \"workspace_id!\",\n j.runnable_path as \"runnable_path\",\n pg_column_size(c.result) as \"result_size_bytes!\",\n c.completed_at as \"completed_at!\"\n FROM (\n SELECT id, workspace_id, result, completed_at\n FROM v2_job_completed\n WHERE completed_at > now() - interval '30 days'\n AND result IS NOT NULL\n ORDER BY completed_at DESC\n LIMIT $1\n ) c\n LEFT JOIN v2_job j ON j.id = c.id\n WHERE pg_column_size(c.result) > 1024\n ORDER BY pg_column_size(c.result) DESC\n LIMIT 10", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id!", + "type_info": "Uuid" + }, + { + "ordinal": 1, + "name": "workspace_id!", + "type_info": "Varchar" + }, + { + "ordinal": 2, + "name": "runnable_path", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "result_size_bytes!", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "completed_at!", + "type_info": "Timestamptz" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + false, + false, + true, + null, + false + ] + }, + "hash": "dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7" +} diff --git a/backend/windmill-api/src/db_health.rs b/backend/windmill-api/src/db_health.rs new file mode 100644 index 0000000000..91c8013950 --- /dev/null +++ b/backend/windmill-api/src/db_health.rs @@ -0,0 +1,512 @@ +/* + * Author: Windmill Labs + * Copyright: Windmill Labs, Inc 2024 + * This file and its contents are licensed under the AGPLv3 License. + * Please see the included NOTICE for copyright information and + * LICENSE-AGPL for a copy of the license. + */ + +use axum::{extract::Query, routing::get, Extension, Json, Router}; +use serde::{Deserialize, Serialize}; + +use windmill_common::error::JsonResult; + +use crate::db::{ApiAuthed, DB}; +use crate::health::get_pool_stats; +use crate::health::PoolStats; +use crate::utils::require_super_admin; + +pub fn global_service() -> Router { + Router::new().route("/", get(get_db_health)) +} + +// --- Response types --- + +#[derive(Serialize)] +#[serde(rename_all = "lowercase")] +pub enum HealthLevel { + Green, + Yellow, + Red, +} + +#[derive(Serialize)] +pub struct DbHealthResponse { + pub database_size: DatabaseSizeInfo, + pub job_retention: JobRetentionInfo, + pub large_results: LargeResultsInfo, + pub connection_pool: ConnectionPoolInfo, + pub table_maintenance: Vec, + pub slow_queries: Option, + pub datatables: Vec, +} + +#[derive(Serialize)] +pub struct DatabaseSizeInfo { + pub total_size_bytes: i64, + pub total_size_pretty: String, + pub top_tables: Vec, +} + +#[derive(Serialize)] +pub struct TableSizeInfo { + pub table_name: String, + pub total_size_bytes: i64, + pub total_size_pretty: String, +} + +#[derive(Serialize)] +pub struct JobRetentionInfo { + pub oldest_completed_at: Option>, + pub total_completed_jobs: i64, + pub retention_period_secs: Option, + pub status: HealthLevel, + pub message: String, +} + +#[derive(Serialize)] +pub struct LargeResultsInfo { + pub top_large_results: Vec, + pub avg_result_size_bytes: Option, +} + +#[derive(Serialize)] +pub struct LargeResultRow { + pub id: uuid::Uuid, + pub workspace_id: String, + pub runnable_path: Option, + pub result_size_bytes: i64, + pub completed_at: chrono::DateTime, +} + +#[derive(Serialize)] +pub struct ConnectionPoolInfo { + pub pool: PoolStats, + pub pg_active_connections: i64, + pub status: HealthLevel, + pub message: String, +} + +#[derive(Serialize)] +pub struct TableMaintenanceInfo { + pub table_name: String, + pub live_tuples: i64, + pub dead_tuples: i64, + pub dead_ratio: f64, + pub last_autovacuum: Option, + pub last_autoanalyze: Option, + pub status: HealthLevel, +} + +#[derive(Serialize)] +pub struct SlowQueriesInfo { + pub queries: Vec, + pub message: Option, +} + +#[derive(Serialize)] +pub struct SlowQueryRow { + pub query: String, + pub calls: i64, + pub total_exec_time_ms: f64, + pub mean_exec_time_ms: f64, +} + +#[derive(Serialize)] +pub struct DatatableInfo { + pub workspace_id: String, + pub name: String, + pub table_name: String, + pub size_bytes: i64, + pub size_pretty: String, + pub estimated_rows: f64, +} + +// --- Handler --- + +#[derive(Deserialize)] +struct DbHealthQuery { + /// Max number of recent completed jobs to scan for large results (default 10000) + scan_limit: Option, +} + +async fn get_db_health( + ApiAuthed { email, .. }: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult { + require_super_admin(&db, &email).await?; + + let scan_limit = query.scan_limit.unwrap_or(10_000).clamp(1_000, 1_000_000); + + let ( + database_size, + job_retention, + large_results, + connection_pool, + table_maintenance, + slow_queries, + datatables, + ) = tokio::try_join!( + fetch_database_size(&db), + fetch_job_retention(&db), + fetch_large_results(&db, scan_limit), + fetch_connection_pool(&db), + fetch_table_maintenance(&db), + fetch_slow_queries(&db), + fetch_datatables(&db), + )?; + + Ok(Json(DbHealthResponse { + database_size, + job_retention, + large_results, + connection_pool, + table_maintenance, + slow_queries, + datatables, + })) +} + +// --- Diagnostic queries --- + +async fn fetch_database_size(db: &DB) -> windmill_common::error::Result { + let row = sqlx::query!( + "SELECT pg_database_size(current_database()) as size_bytes, pg_size_pretty(pg_database_size(current_database())) as size_pretty" + ) + .fetch_one(db) + .await?; + + let top_tables = sqlx::query_as!( + TableSizeInfo, + r#"SELECT + schemaname || '.' || relname as "table_name!", + pg_total_relation_size(relid) as "total_size_bytes!", + pg_size_pretty(pg_total_relation_size(relid)) as "total_size_pretty!" + FROM pg_catalog.pg_statio_user_tables + ORDER BY pg_total_relation_size(relid) DESC + LIMIT 15"# + ) + .fetch_all(db) + .await?; + + Ok(DatabaseSizeInfo { + total_size_bytes: row.size_bytes.unwrap_or(0), + total_size_pretty: row.size_pretty.unwrap_or_default(), + top_tables, + }) +} + +async fn fetch_job_retention(db: &DB) -> windmill_common::error::Result { + let job_row = + sqlx::query!("SELECT MIN(completed_at) as oldest, COUNT(*) as total FROM v2_job_completed") + .fetch_one(db) + .await?; + + let retention_row = + sqlx::query!("SELECT value FROM global_settings WHERE name = 'retention_period_secs'") + .fetch_optional(db) + .await?; + + let retention_period_secs: Option = + retention_row.map(|r| r.value).and_then(|v| v.as_i64()); + + let oldest = job_row.oldest; + let total = job_row.total.unwrap_or(0); + + let (status, message) = if let (Some(oldest_ts), Some(retention_secs)) = + (oldest, retention_period_secs) + { + let age_secs: i64 = (chrono::Utc::now() - oldest_ts).num_seconds(); + let ratio = if retention_secs > 0 { + age_secs as f64 / retention_secs as f64 + } else { + 0.0 + }; + if ratio <= 2.0 { + ( + HealthLevel::Green, + format!( + "Oldest job is {:.1}x the retention period. Cleanup is keeping up.", + ratio + ), + ) + } else if ratio <= 5.0 { + ( + HealthLevel::Yellow, + format!( + "Oldest job is {:.1}x the retention period. Cleanup may be falling behind.", + ratio + ), + ) + } else { + (HealthLevel::Red, format!("Oldest job is {:.1}x the retention period. Consider reducing retention or investigating cleanup.", ratio)) + } + } else if oldest.is_some() && retention_period_secs.is_none() { + ( + HealthLevel::Yellow, + "No retention_period_secs configured. Old jobs will accumulate.".to_string(), + ) + } else { + (HealthLevel::Green, "No completed jobs found.".to_string()) + }; + + Ok(JobRetentionInfo { + oldest_completed_at: oldest, + total_completed_jobs: total, + retention_period_secs, + status, + message, + }) +} + +async fn fetch_large_results( + db: &DB, + scan_limit: i64, +) -> windmill_common::error::Result { + let top_large_results = sqlx::query_as!( + LargeResultRow, + r#"SELECT + c.id as "id!", + c.workspace_id as "workspace_id!", + j.runnable_path as "runnable_path", + pg_column_size(c.result) as "result_size_bytes!", + c.completed_at as "completed_at!" + FROM ( + SELECT id, workspace_id, result, completed_at + FROM v2_job_completed + WHERE completed_at > now() - interval '30 days' + AND result IS NOT NULL + ORDER BY completed_at DESC + LIMIT $1 + ) c + LEFT JOIN v2_job j ON j.id = c.id + WHERE pg_column_size(c.result) > 1024 + ORDER BY pg_column_size(c.result) DESC + LIMIT 10"#, + scan_limit + ) + .fetch_all(db) + .await?; + + let avg_row = sqlx::query!( + r#"SELECT AVG(pg_column_size(result))::bigint as "avg_size" + FROM ( + SELECT result FROM v2_job_completed + WHERE completed_at > now() - interval '30 days' + AND result IS NOT NULL + ORDER BY completed_at DESC + LIMIT $1 + ) sub"#, + scan_limit + ) + .fetch_one(db) + .await?; + + Ok(LargeResultsInfo { top_large_results, avg_result_size_bytes: avg_row.avg_size }) +} + +async fn fetch_connection_pool(db: &DB) -> windmill_common::error::Result { + let pool = get_pool_stats(db); + + let active_row = + sqlx::query!("SELECT COUNT(*) as cnt FROM pg_stat_activity WHERE state = 'active'") + .fetch_one(db) + .await?; + + let pg_active = active_row.cnt.unwrap_or(0); + let utilization = if pool.max_connections > 0 { + pool.size as f64 / pool.max_connections as f64 + } else { + 0.0 + }; + + let (status, message) = if utilization < 0.8 { + ( + HealthLevel::Green, + format!( + "Pool utilization: {:.0}% ({}/{})", + utilization * 100.0, + pool.size, + pool.max_connections + ), + ) + } else if utilization < 0.95 { + ( + HealthLevel::Yellow, + format!( + "Pool utilization is high: {:.0}% ({}/{}). Consider increasing max_connections.", + utilization * 100.0, + pool.size, + pool.max_connections + ), + ) + } else { + ( + HealthLevel::Red, + format!( + "Pool near exhaustion: {:.0}% ({}/{}). Increase max_connections urgently.", + utilization * 100.0, + pool.size, + pool.max_connections + ), + ) + }; + + Ok(ConnectionPoolInfo { pool, pg_active_connections: pg_active, status, message }) +} + +async fn fetch_table_maintenance( + db: &DB, +) -> windmill_common::error::Result> { + let rows = sqlx::query!( + r#"SELECT + schemaname || '.' || relname as "table_name!", + COALESCE(n_live_tup, 0) as "live_tuples!", + COALESCE(n_dead_tup, 0) as "dead_tuples!", + last_autovacuum as "last_autovacuum", + last_autoanalyze as "last_autoanalyze" + FROM pg_stat_user_tables + ORDER BY n_dead_tup DESC + LIMIT 15"# + ) + .fetch_all(db) + .await?; + + Ok(rows + .into_iter() + .map(|r| { + let total = r.live_tuples + r.dead_tuples; + let dead_ratio = if total > 0 { + r.dead_tuples as f64 / total as f64 + } else { + 0.0 + }; + let status = if dead_ratio < 0.1 { + HealthLevel::Green + } else if dead_ratio < 0.3 { + HealthLevel::Yellow + } else { + HealthLevel::Red + }; + TableMaintenanceInfo { + table_name: r.table_name, + live_tuples: r.live_tuples, + dead_tuples: r.dead_tuples, + dead_ratio, + last_autovacuum: r.last_autovacuum.map(|t| t.naive_utc()), + last_autoanalyze: r.last_autoanalyze.map(|t| t.naive_utc()), + status, + } + }) + .collect()) +} + +async fn fetch_slow_queries(db: &DB) -> windmill_common::error::Result> { + let ext_exists: bool = sqlx::query_scalar!( + r#"SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') as "exists!""# + ) + .fetch_one(db) + .await?; + + if !ext_exists { + return Ok(Some(SlowQueriesInfo { + queries: vec![], + message: Some( + "pg_stat_statements extension is not installed. Enable it for slow query insights." + .to_string(), + ), + })); + } + + // Use raw query since pg_stat_statements may not exist at compile time + let rows: Vec = sqlx::query_as::<_, (String, i64, f64, f64)>( + r#"SELECT + LEFT(query, 200), + calls::bigint, + total_exec_time::float8, + mean_exec_time::float8 + FROM pg_stat_statements + WHERE query NOT LIKE '%pg_stat_statements%' + ORDER BY mean_exec_time DESC + LIMIT 10"#, + ) + .fetch_all(db) + .await? + .into_iter() + .map( + |(query, calls, total_exec_time_ms, mean_exec_time_ms)| SlowQueryRow { + query, + calls, + total_exec_time_ms, + mean_exec_time_ms, + }, + ) + .collect(); + + Ok(Some(SlowQueriesInfo { queries: rows, message: None })) +} + +async fn fetch_datatables(db: &DB) -> windmill_common::error::Result> { + // Find instance-type datatables from workspace_settings + let rows = sqlx::query!( + r#"SELECT + ws.workspace_id as "workspace_id!", + dt.key as "name!", + dt.value->>'table_name' as "table_name" + FROM workspace_settings ws, + jsonb_each(ws.datatable) dt + WHERE dt.value->>'resource_type' = 'instance' + AND dt.value->>'table_name' IS NOT NULL"# + ) + .fetch_all(db) + .await?; + + let table_names: Vec = rows.iter().filter_map(|r| r.table_name.clone()).collect(); + + if table_names.is_empty() { + return Ok(vec![]); + } + + // Batch lookup: single query for all table sizes + let size_rows = sqlx::query!( + r#"SELECT + c.relname as "table_name!", + pg_total_relation_size(c.oid) as "size_bytes!", + pg_size_pretty(pg_total_relation_size(c.oid)) as "size_pretty!", + COALESCE(c.reltuples, 0) as "estimated_rows!" + FROM pg_class c + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = ANY($1)"#, + &table_names + ) + .fetch_all(db) + .await?; + + let size_map: std::collections::HashMap = size_rows + .into_iter() + .map(|s| (s.table_name.clone(), s)) + .collect(); + + let mut result = Vec::new(); + for row in rows { + let table_name = match &row.table_name { + Some(t) => t.clone(), + None => continue, + }; + if let Some(s) = size_map.get(&table_name) { + result.push(DatatableInfo { + workspace_id: row.workspace_id, + name: row.name, + table_name, + size_bytes: s.size_bytes, + size_pretty: s.size_pretty.clone(), + estimated_rows: s.estimated_rows as f64, + }); + } + } + + // Sort by size descending + result.sort_by(|a, b| b.size_bytes.cmp(&a.size_bytes)); + Ok(result) +} diff --git a/backend/windmill-api/src/health.rs b/backend/windmill-api/src/health.rs index c31b104429..fab9979c12 100644 --- a/backend/windmill-api/src/health.rs +++ b/backend/windmill-api/src/health.rs @@ -231,7 +231,7 @@ async fn check_database_with_latency(db: &DB) -> DatabaseCheckResult { DatabaseCheckResult { healthy, latency_ms } } -fn get_pool_stats(db: &DB) -> PoolStats { +pub(crate) fn get_pool_stats(db: &DB) -> PoolStats { PoolStats { size: db.size(), idle: db.num_idle() as u32, diff --git a/backend/windmill-api/src/lib.rs b/backend/windmill-api/src/lib.rs index e714689df0..3491e50c69 100644 --- a/backend/windmill-api/src/lib.rs +++ b/backend/windmill-api/src/lib.rs @@ -77,6 +77,7 @@ mod bedrock; mod capture; mod concurrency_groups; mod db; +mod db_health; mod google; mod drafts; @@ -645,6 +646,7 @@ pub async fn run_server( .nest("/inkeep", inkeep_oss::global_service()) .nest("/indexer", indexer_oss::management_service()) .nest("/mcp/w/{workspace_id}/list_tools", mcp_list_tools_service) + .nest("/db_health", db_health::global_service()) .nest("/health/detailed", health::detailed_service()) .nest( "/saml", diff --git a/frontend/src/lib/components/InstanceSettings.svelte b/frontend/src/lib/components/InstanceSettings.svelte index 18514b3836..d558910aad 100644 --- a/frontend/src/lib/components/InstanceSettings.svelte +++ b/frontend/src/lib/components/InstanceSettings.svelte @@ -21,6 +21,7 @@ import SettingsFooter from './workspaceSettings/SettingsFooter.svelte' import SettingsPageHeader from './settings/SettingsPageHeader.svelte' import WorkspaceRegistries from './instanceSettings/WorkspaceRegistries.svelte' + import DbHealth from './instanceSettings/DbHealth.svelte' interface Props { tab?: string @@ -1052,6 +1053,12 @@ title="GitHub Enterprise App" description="Configure a self-managed GitHub App for GitHub Enterprise Server git sync." /> + {:else if category == 'DB Health'} + + {:else if category == 'Auth/OAuth/SAML'} = { } ], 'Auth/OAuth/SAML': [], + 'DB Health': [], Registries: [ { label: 'Instance Python Version', @@ -829,6 +830,12 @@ export const instanceSettingsNavigationGroups = [ aiId: 'instance-settings-indexer', aiDescription: 'Instance indexer settings', isEE: true + }, + { + id: 'db_health', + label: 'DB Health', + aiId: 'instance-settings-db-health', + aiDescription: 'Database health diagnostics and performance insights' } ] }, @@ -900,7 +907,8 @@ export const tabToCategoryMap: Record = { jobs: 'Jobs', private_hub: 'Private Hub', github_enterprise_app: 'GitHub App', - websocket: 'WebSocket' + websocket: 'WebSocket', + db_health: 'DB Health' } export const tabToAuthSubTab: Record = { @@ -933,7 +941,8 @@ export const categoryToTabMap: Record = { Jobs: 'jobs', 'Private Hub': 'private_hub', 'GitHub App': 'github_enterprise_app', - WebSocket: 'websocket' + WebSocket: 'websocket', + 'DB Health': 'db_health' } export interface SearchableSettingItem { diff --git a/frontend/src/lib/components/instanceSettings/DbHealth.svelte b/frontend/src/lib/components/instanceSettings/DbHealth.svelte new file mode 100644 index 0000000000..df1cfa7673 --- /dev/null +++ b/frontend/src/lib/components/instanceSettings/DbHealth.svelte @@ -0,0 +1,502 @@ + + +
+
+ + + {#if loading} + This may take a few seconds... + {/if} +
+ + {#if error} +
+ {error} +
+ {/if} + + {#if data} + +
+ + {#if expandedSections.database_size} +
+

+ Total database size: {data.database_size.total_size_pretty} +

+
+ + + + + + + + + {#each data.database_size.top_tables as t} + + + + + {/each} + +
TableSize
{t.table_name}{t.total_size_pretty}
+
+
+ {/if} +
+ + +
+ + {#if expandedSections.job_retention} +
+

+ Total completed jobs: {formatNumber(data.job_retention.total_completed_jobs)} +

+

+ Oldest job: {formatDate(data.job_retention.oldest_completed_at)} +

+

+ Retention period: + {data.job_retention.retention_period_secs + ? formatNumber(data.job_retention.retention_period_secs) + 's' + : 'Not configured'} + +

+

+ {data.job_retention.message} +

+
+ {/if} +
+ + +
+ + {#if expandedSections.large_results} +
+ {#if data.large_results.top_large_results.length === 0} +

No job results found in the last 30 days.

+ {:else} +
+ + + + + + + + + + + + {#each data.large_results.top_large_results as r} + + + + + + + + {/each} + +
Job IDWorkspaceScriptResult SizeCompleted
{r.id.substring(0, 8)}...{r.workspace_id}{r.runnable_path ?? '-'}{formatBytes(r.result_size_bytes)}{formatDate(r.completed_at)}
+
+ {/if} +
+ {/if} +
+ + +
+ + {#if expandedSections.connection_pool} +
+

+ Pool size: {data.connection_pool.pool.size} / Max: + {data.connection_pool.pool.max_connections} + / Idle: {data.connection_pool.pool.idle} +

+

+ Active PG connections: {data.connection_pool.pg_active_connections} +

+

+ {data.connection_pool.message} +

+
+ {/if} +
+ + +
+ + {#if expandedSections.table_maintenance} +
+
+ + + + + + + + + + + + + + {#each data.table_maintenance as t} + + + + + + + + + + {/each} + +
TableLive TuplesDead TuplesDead %Last VacuumLast AnalyzeStatus
{t.table_name}{formatNumber(t.live_tuples)}{formatNumber(t.dead_tuples)}{(t.dead_ratio * 100).toFixed(1)}%{formatDate(t.last_autovacuum)}{formatDate(t.last_autoanalyze)} + + {t.status} + +
+
+
+ {/if} +
+ + +
+ + {#if expandedSections.slow_queries} +
+ {#if data.slow_queries == null} +

Slow query data not available.

+ {:else if data.slow_queries.message} +

{data.slow_queries.message}

+ {:else if data.slow_queries.queries.length === 0} +

No slow queries found.

+ {:else} +
+ + + + + + + + + + + {#each data.slow_queries.queries as q} + + + + + + + {/each} + +
QueryCallsTotal TimeMean Time
{q.query}{formatNumber(q.calls)}{formatMs(q.total_exec_time_ms)}{formatMs(q.mean_exec_time_ms)}
+
+ {/if} +
+ {/if} +
+ + +
+ + {#if expandedSections.datatables} +
+ {#if data.datatables.length === 0} +

No instance-stored datatables found.

+ {:else} +
+ + + + + + + + + + + + {#each data.datatables as dt} + + + + + + + + {/each} + +
WorkspaceNameTableSizeEst. Rows
{dt.workspace_id}{dt.name}{dt.table_name}{dt.size_pretty}{formatNumber(Math.round(dt.estimated_rows))}
+
+ {/if} +
+ {/if} +
+ {:else if !loading} +

+ Click "Run Diagnostics" to analyze your database health. The queries are read-only and + lightweight. +

+ {/if} +