feat: add DB health diagnostic dashboard for superadmins (#8574)

* feat: add DB health diagnostic dashboard for superadmins

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* Update SQLx metadata

* fix: improve db health query performance

Bound large_results scan to last N jobs (configurable via scan_limit
query param, default 10K) instead of full-table pg_column_size sort.
Replace N+1 datatable size queries with single batched pg_class lookup.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* Update SQLx metadata

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* sqlx

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: windmill-internal-app[bot] <windmill-internal-app[bot]@users.noreply.github.com>
Co-authored-by: Ruben Fiszel <ruben@windmill.dev>
This commit is contained in:
Alexander Petric
2026-03-28 05:32:10 -04:00
committed by GitHub
parent d29cb234db
commit 9ceab730d7
17 changed files with 1363 additions and 3 deletions
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') as \"exists!\"",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "exists!",
"type_info": "Bool"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "143acebe5d815c5d828013ebe46274f891f953c75f821499552ab7794f75063d"
}
@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n schemaname || '.' || relname as \"table_name!\",\n pg_total_relation_size(relid) as \"total_size_bytes!\",\n pg_size_pretty(pg_total_relation_size(relid)) as \"total_size_pretty!\"\n FROM pg_catalog.pg_statio_user_tables\n ORDER BY pg_total_relation_size(relid) DESC\n LIMIT 15",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "table_name!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "total_size_bytes!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "total_size_pretty!",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null,
null
]
},
"hash": "1dd73eff0e89b84c0316af2760a136afdd19dc34f9f31c4f9de6b0f74bc386a6"
}
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT value FROM global_settings WHERE name = 'retention_period_secs'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "value",
"type_info": "Jsonb"
}
],
"parameters": {
"Left": []
},
"nullable": [
false
]
},
"hash": "26e62b4509e44a7548957ad4ef217fd46bc03d5dca19344cd3bf7b131fa40ed2"
}
@@ -0,0 +1,32 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n ws.workspace_id as \"workspace_id!\",\n dt.key as \"name!\",\n dt.value->>'table_name' as \"table_name\"\n FROM workspace_settings ws,\n jsonb_each(ws.datatable) dt\n WHERE dt.value->>'resource_type' = 'instance'\n AND dt.value->>'table_name' IS NOT NULL",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 1,
"name": "name!",
"type_info": "Text"
},
{
"ordinal": 2,
"name": "table_name",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
false,
null,
null
]
},
"hash": "2d4ccf3ee19a70cbb5bd034c74703bbb30f217cd3673821e11bae3bf9f925720"
}
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT pg_database_size(current_database()) as size_bytes, pg_size_pretty(pg_database_size(current_database())) as size_pretty",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "size_bytes",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "size_pretty",
"type_info": "Text"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "384f5e9b2ab8e430141e28ea58854cbcfbcf96fd2adbf0513ce942cfe9bceaf0"
}
@@ -0,0 +1,40 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n c.relname as \"table_name!\",\n pg_total_relation_size(c.oid) as \"size_bytes!\",\n pg_size_pretty(pg_total_relation_size(c.oid)) as \"size_pretty!\",\n COALESCE(c.reltuples, 0) as \"estimated_rows!\"\n FROM pg_class c\n JOIN pg_namespace n ON n.oid = c.relnamespace\n WHERE n.nspname = 'public' AND c.relname = ANY($1)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "table_name!",
"type_info": "Name"
},
{
"ordinal": 1,
"name": "size_bytes!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "size_pretty!",
"type_info": "Text"
},
{
"ordinal": 3,
"name": "estimated_rows!",
"type_info": "Float4"
}
],
"parameters": {
"Left": [
"NameArray"
]
},
"nullable": [
false,
null,
null,
null
]
},
"hash": "7c5db0b3bd1dd1f766e1841ca620871a468033e05b6e0188ea4775b63fc66e84"
}
@@ -0,0 +1,20 @@
{
"db_name": "PostgreSQL",
"query": "SELECT COUNT(*) as cnt FROM pg_stat_activity WHERE state = 'active'",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "cnt",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "87d07998fe8373f5b89eccf6f0528c02e389bf827d935d867430ad3459104dd9"
}
@@ -0,0 +1,22 @@
{
"db_name": "PostgreSQL",
"query": "SELECT AVG(pg_column_size(result))::bigint as \"avg_size\"\n FROM (\n SELECT result FROM v2_job_completed\n WHERE completed_at > now() - interval '30 days'\n AND result IS NOT NULL\n ORDER BY completed_at DESC\n LIMIT $1\n ) sub",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "avg_size",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
null
]
},
"hash": "a9c3461ca3053f699c957f61780d1e889ad53dc5bf1669c24c0666c290656c00"
}
@@ -0,0 +1,26 @@
{
"db_name": "PostgreSQL",
"query": "SELECT MIN(completed_at) as oldest, COUNT(*) as total FROM v2_job_completed",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "oldest",
"type_info": "Timestamptz"
},
{
"ordinal": 1,
"name": "total",
"type_info": "Int8"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null
]
},
"hash": "b760be4a0a80853073a061f7c9ebc2d411294d57b07d54d15d178db3c6ee2a30"
}
@@ -0,0 +1,44 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n schemaname || '.' || relname as \"table_name!\",\n COALESCE(n_live_tup, 0) as \"live_tuples!\",\n COALESCE(n_dead_tup, 0) as \"dead_tuples!\",\n last_autovacuum as \"last_autovacuum\",\n last_autoanalyze as \"last_autoanalyze\"\n FROM pg_stat_user_tables\n ORDER BY n_dead_tup DESC\n LIMIT 15",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "table_name!",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "live_tuples!",
"type_info": "Int8"
},
{
"ordinal": 2,
"name": "dead_tuples!",
"type_info": "Int8"
},
{
"ordinal": 3,
"name": "last_autovacuum",
"type_info": "Timestamptz"
},
{
"ordinal": 4,
"name": "last_autoanalyze",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": []
},
"nullable": [
null,
null,
null,
true,
true
]
},
"hash": "bc54ea311580a0525c1f36aaa543c5798e6f7aca1e6e564330766d77038ef0e3"
}
@@ -0,0 +1,46 @@
{
"db_name": "PostgreSQL",
"query": "SELECT\n c.id as \"id!\",\n c.workspace_id as \"workspace_id!\",\n j.runnable_path as \"runnable_path\",\n pg_column_size(c.result) as \"result_size_bytes!\",\n c.completed_at as \"completed_at!\"\n FROM (\n SELECT id, workspace_id, result, completed_at\n FROM v2_job_completed\n WHERE completed_at > now() - interval '30 days'\n AND result IS NOT NULL\n ORDER BY completed_at DESC\n LIMIT $1\n ) c\n LEFT JOIN v2_job j ON j.id = c.id\n WHERE pg_column_size(c.result) > 1024\n ORDER BY pg_column_size(c.result) DESC\n LIMIT 10",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id!",
"type_info": "Uuid"
},
{
"ordinal": 1,
"name": "workspace_id!",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "runnable_path",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "result_size_bytes!",
"type_info": "Int4"
},
{
"ordinal": 4,
"name": "completed_at!",
"type_info": "Timestamptz"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false,
false,
true,
null,
false
]
},
"hash": "dbc5924bca3aa0b32e296b73f8a967bed68332caf526216597f10ffa5fa951c7"
}
+512
View File
@@ -0,0 +1,512 @@
/*
* Author: Windmill Labs
* Copyright: Windmill Labs, Inc 2024
* This file and its contents are licensed under the AGPLv3 License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use axum::{extract::Query, routing::get, Extension, Json, Router};
use serde::{Deserialize, Serialize};
use windmill_common::error::JsonResult;
use crate::db::{ApiAuthed, DB};
use crate::health::get_pool_stats;
use crate::health::PoolStats;
use crate::utils::require_super_admin;
pub fn global_service() -> Router {
Router::new().route("/", get(get_db_health))
}
// --- Response types ---
#[derive(Serialize)]
#[serde(rename_all = "lowercase")]
pub enum HealthLevel {
Green,
Yellow,
Red,
}
#[derive(Serialize)]
pub struct DbHealthResponse {
pub database_size: DatabaseSizeInfo,
pub job_retention: JobRetentionInfo,
pub large_results: LargeResultsInfo,
pub connection_pool: ConnectionPoolInfo,
pub table_maintenance: Vec<TableMaintenanceInfo>,
pub slow_queries: Option<SlowQueriesInfo>,
pub datatables: Vec<DatatableInfo>,
}
#[derive(Serialize)]
pub struct DatabaseSizeInfo {
pub total_size_bytes: i64,
pub total_size_pretty: String,
pub top_tables: Vec<TableSizeInfo>,
}
#[derive(Serialize)]
pub struct TableSizeInfo {
pub table_name: String,
pub total_size_bytes: i64,
pub total_size_pretty: String,
}
#[derive(Serialize)]
pub struct JobRetentionInfo {
pub oldest_completed_at: Option<chrono::DateTime<chrono::Utc>>,
pub total_completed_jobs: i64,
pub retention_period_secs: Option<i64>,
pub status: HealthLevel,
pub message: String,
}
#[derive(Serialize)]
pub struct LargeResultsInfo {
pub top_large_results: Vec<LargeResultRow>,
pub avg_result_size_bytes: Option<i64>,
}
#[derive(Serialize)]
pub struct LargeResultRow {
pub id: uuid::Uuid,
pub workspace_id: String,
pub runnable_path: Option<String>,
pub result_size_bytes: i64,
pub completed_at: chrono::DateTime<chrono::Utc>,
}
#[derive(Serialize)]
pub struct ConnectionPoolInfo {
pub pool: PoolStats,
pub pg_active_connections: i64,
pub status: HealthLevel,
pub message: String,
}
#[derive(Serialize)]
pub struct TableMaintenanceInfo {
pub table_name: String,
pub live_tuples: i64,
pub dead_tuples: i64,
pub dead_ratio: f64,
pub last_autovacuum: Option<chrono::NaiveDateTime>,
pub last_autoanalyze: Option<chrono::NaiveDateTime>,
pub status: HealthLevel,
}
#[derive(Serialize)]
pub struct SlowQueriesInfo {
pub queries: Vec<SlowQueryRow>,
pub message: Option<String>,
}
#[derive(Serialize)]
pub struct SlowQueryRow {
pub query: String,
pub calls: i64,
pub total_exec_time_ms: f64,
pub mean_exec_time_ms: f64,
}
#[derive(Serialize)]
pub struct DatatableInfo {
pub workspace_id: String,
pub name: String,
pub table_name: String,
pub size_bytes: i64,
pub size_pretty: String,
pub estimated_rows: f64,
}
// --- Handler ---
#[derive(Deserialize)]
struct DbHealthQuery {
/// Max number of recent completed jobs to scan for large results (default 10000)
scan_limit: Option<i64>,
}
async fn get_db_health(
ApiAuthed { email, .. }: ApiAuthed,
Extension(db): Extension<DB>,
Query(query): Query<DbHealthQuery>,
) -> JsonResult<DbHealthResponse> {
require_super_admin(&db, &email).await?;
let scan_limit = query.scan_limit.unwrap_or(10_000).clamp(1_000, 1_000_000);
let (
database_size,
job_retention,
large_results,
connection_pool,
table_maintenance,
slow_queries,
datatables,
) = tokio::try_join!(
fetch_database_size(&db),
fetch_job_retention(&db),
fetch_large_results(&db, scan_limit),
fetch_connection_pool(&db),
fetch_table_maintenance(&db),
fetch_slow_queries(&db),
fetch_datatables(&db),
)?;
Ok(Json(DbHealthResponse {
database_size,
job_retention,
large_results,
connection_pool,
table_maintenance,
slow_queries,
datatables,
}))
}
// --- Diagnostic queries ---
async fn fetch_database_size(db: &DB) -> windmill_common::error::Result<DatabaseSizeInfo> {
let row = sqlx::query!(
"SELECT pg_database_size(current_database()) as size_bytes, pg_size_pretty(pg_database_size(current_database())) as size_pretty"
)
.fetch_one(db)
.await?;
let top_tables = sqlx::query_as!(
TableSizeInfo,
r#"SELECT
schemaname || '.' || relname as "table_name!",
pg_total_relation_size(relid) as "total_size_bytes!",
pg_size_pretty(pg_total_relation_size(relid)) as "total_size_pretty!"
FROM pg_catalog.pg_statio_user_tables
ORDER BY pg_total_relation_size(relid) DESC
LIMIT 15"#
)
.fetch_all(db)
.await?;
Ok(DatabaseSizeInfo {
total_size_bytes: row.size_bytes.unwrap_or(0),
total_size_pretty: row.size_pretty.unwrap_or_default(),
top_tables,
})
}
async fn fetch_job_retention(db: &DB) -> windmill_common::error::Result<JobRetentionInfo> {
let job_row =
sqlx::query!("SELECT MIN(completed_at) as oldest, COUNT(*) as total FROM v2_job_completed")
.fetch_one(db)
.await?;
let retention_row =
sqlx::query!("SELECT value FROM global_settings WHERE name = 'retention_period_secs'")
.fetch_optional(db)
.await?;
let retention_period_secs: Option<i64> =
retention_row.map(|r| r.value).and_then(|v| v.as_i64());
let oldest = job_row.oldest;
let total = job_row.total.unwrap_or(0);
let (status, message) = if let (Some(oldest_ts), Some(retention_secs)) =
(oldest, retention_period_secs)
{
let age_secs: i64 = (chrono::Utc::now() - oldest_ts).num_seconds();
let ratio = if retention_secs > 0 {
age_secs as f64 / retention_secs as f64
} else {
0.0
};
if ratio <= 2.0 {
(
HealthLevel::Green,
format!(
"Oldest job is {:.1}x the retention period. Cleanup is keeping up.",
ratio
),
)
} else if ratio <= 5.0 {
(
HealthLevel::Yellow,
format!(
"Oldest job is {:.1}x the retention period. Cleanup may be falling behind.",
ratio
),
)
} else {
(HealthLevel::Red, format!("Oldest job is {:.1}x the retention period. Consider reducing retention or investigating cleanup.", ratio))
}
} else if oldest.is_some() && retention_period_secs.is_none() {
(
HealthLevel::Yellow,
"No retention_period_secs configured. Old jobs will accumulate.".to_string(),
)
} else {
(HealthLevel::Green, "No completed jobs found.".to_string())
};
Ok(JobRetentionInfo {
oldest_completed_at: oldest,
total_completed_jobs: total,
retention_period_secs,
status,
message,
})
}
async fn fetch_large_results(
db: &DB,
scan_limit: i64,
) -> windmill_common::error::Result<LargeResultsInfo> {
let top_large_results = sqlx::query_as!(
LargeResultRow,
r#"SELECT
c.id as "id!",
c.workspace_id as "workspace_id!",
j.runnable_path as "runnable_path",
pg_column_size(c.result) as "result_size_bytes!",
c.completed_at as "completed_at!"
FROM (
SELECT id, workspace_id, result, completed_at
FROM v2_job_completed
WHERE completed_at > now() - interval '30 days'
AND result IS NOT NULL
ORDER BY completed_at DESC
LIMIT $1
) c
LEFT JOIN v2_job j ON j.id = c.id
WHERE pg_column_size(c.result) > 1024
ORDER BY pg_column_size(c.result) DESC
LIMIT 10"#,
scan_limit
)
.fetch_all(db)
.await?;
let avg_row = sqlx::query!(
r#"SELECT AVG(pg_column_size(result))::bigint as "avg_size"
FROM (
SELECT result FROM v2_job_completed
WHERE completed_at > now() - interval '30 days'
AND result IS NOT NULL
ORDER BY completed_at DESC
LIMIT $1
) sub"#,
scan_limit
)
.fetch_one(db)
.await?;
Ok(LargeResultsInfo { top_large_results, avg_result_size_bytes: avg_row.avg_size })
}
async fn fetch_connection_pool(db: &DB) -> windmill_common::error::Result<ConnectionPoolInfo> {
let pool = get_pool_stats(db);
let active_row =
sqlx::query!("SELECT COUNT(*) as cnt FROM pg_stat_activity WHERE state = 'active'")
.fetch_one(db)
.await?;
let pg_active = active_row.cnt.unwrap_or(0);
let utilization = if pool.max_connections > 0 {
pool.size as f64 / pool.max_connections as f64
} else {
0.0
};
let (status, message) = if utilization < 0.8 {
(
HealthLevel::Green,
format!(
"Pool utilization: {:.0}% ({}/{})",
utilization * 100.0,
pool.size,
pool.max_connections
),
)
} else if utilization < 0.95 {
(
HealthLevel::Yellow,
format!(
"Pool utilization is high: {:.0}% ({}/{}). Consider increasing max_connections.",
utilization * 100.0,
pool.size,
pool.max_connections
),
)
} else {
(
HealthLevel::Red,
format!(
"Pool near exhaustion: {:.0}% ({}/{}). Increase max_connections urgently.",
utilization * 100.0,
pool.size,
pool.max_connections
),
)
};
Ok(ConnectionPoolInfo { pool, pg_active_connections: pg_active, status, message })
}
async fn fetch_table_maintenance(
db: &DB,
) -> windmill_common::error::Result<Vec<TableMaintenanceInfo>> {
let rows = sqlx::query!(
r#"SELECT
schemaname || '.' || relname as "table_name!",
COALESCE(n_live_tup, 0) as "live_tuples!",
COALESCE(n_dead_tup, 0) as "dead_tuples!",
last_autovacuum as "last_autovacuum",
last_autoanalyze as "last_autoanalyze"
FROM pg_stat_user_tables
ORDER BY n_dead_tup DESC
LIMIT 15"#
)
.fetch_all(db)
.await?;
Ok(rows
.into_iter()
.map(|r| {
let total = r.live_tuples + r.dead_tuples;
let dead_ratio = if total > 0 {
r.dead_tuples as f64 / total as f64
} else {
0.0
};
let status = if dead_ratio < 0.1 {
HealthLevel::Green
} else if dead_ratio < 0.3 {
HealthLevel::Yellow
} else {
HealthLevel::Red
};
TableMaintenanceInfo {
table_name: r.table_name,
live_tuples: r.live_tuples,
dead_tuples: r.dead_tuples,
dead_ratio,
last_autovacuum: r.last_autovacuum.map(|t| t.naive_utc()),
last_autoanalyze: r.last_autoanalyze.map(|t| t.naive_utc()),
status,
}
})
.collect())
}
async fn fetch_slow_queries(db: &DB) -> windmill_common::error::Result<Option<SlowQueriesInfo>> {
let ext_exists: bool = sqlx::query_scalar!(
r#"SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') as "exists!""#
)
.fetch_one(db)
.await?;
if !ext_exists {
return Ok(Some(SlowQueriesInfo {
queries: vec![],
message: Some(
"pg_stat_statements extension is not installed. Enable it for slow query insights."
.to_string(),
),
}));
}
// Use raw query since pg_stat_statements may not exist at compile time
let rows: Vec<SlowQueryRow> = sqlx::query_as::<_, (String, i64, f64, f64)>(
r#"SELECT
LEFT(query, 200),
calls::bigint,
total_exec_time::float8,
mean_exec_time::float8
FROM pg_stat_statements
WHERE query NOT LIKE '%pg_stat_statements%'
ORDER BY mean_exec_time DESC
LIMIT 10"#,
)
.fetch_all(db)
.await?
.into_iter()
.map(
|(query, calls, total_exec_time_ms, mean_exec_time_ms)| SlowQueryRow {
query,
calls,
total_exec_time_ms,
mean_exec_time_ms,
},
)
.collect();
Ok(Some(SlowQueriesInfo { queries: rows, message: None }))
}
async fn fetch_datatables(db: &DB) -> windmill_common::error::Result<Vec<DatatableInfo>> {
// Find instance-type datatables from workspace_settings
let rows = sqlx::query!(
r#"SELECT
ws.workspace_id as "workspace_id!",
dt.key as "name!",
dt.value->>'table_name' as "table_name"
FROM workspace_settings ws,
jsonb_each(ws.datatable) dt
WHERE dt.value->>'resource_type' = 'instance'
AND dt.value->>'table_name' IS NOT NULL"#
)
.fetch_all(db)
.await?;
let table_names: Vec<String> = rows.iter().filter_map(|r| r.table_name.clone()).collect();
if table_names.is_empty() {
return Ok(vec![]);
}
// Batch lookup: single query for all table sizes
let size_rows = sqlx::query!(
r#"SELECT
c.relname as "table_name!",
pg_total_relation_size(c.oid) as "size_bytes!",
pg_size_pretty(pg_total_relation_size(c.oid)) as "size_pretty!",
COALESCE(c.reltuples, 0) as "estimated_rows!"
FROM pg_class c
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = 'public' AND c.relname = ANY($1)"#,
&table_names
)
.fetch_all(db)
.await?;
let size_map: std::collections::HashMap<String, _> = size_rows
.into_iter()
.map(|s| (s.table_name.clone(), s))
.collect();
let mut result = Vec::new();
for row in rows {
let table_name = match &row.table_name {
Some(t) => t.clone(),
None => continue,
};
if let Some(s) = size_map.get(&table_name) {
result.push(DatatableInfo {
workspace_id: row.workspace_id,
name: row.name,
table_name,
size_bytes: s.size_bytes,
size_pretty: s.size_pretty.clone(),
estimated_rows: s.estimated_rows as f64,
});
}
}
// Sort by size descending
result.sort_by(|a, b| b.size_bytes.cmp(&a.size_bytes));
Ok(result)
}
+1 -1
View File
@@ -231,7 +231,7 @@ async fn check_database_with_latency(db: &DB) -> DatabaseCheckResult {
DatabaseCheckResult { healthy, latency_ms }
}
fn get_pool_stats(db: &DB) -> PoolStats {
pub(crate) fn get_pool_stats(db: &DB) -> PoolStats {
PoolStats {
size: db.size(),
idle: db.num_idle() as u32,
+2
View File
@@ -77,6 +77,7 @@ mod bedrock;
mod capture;
mod concurrency_groups;
mod db;
mod db_health;
mod google;
mod drafts;
@@ -645,6 +646,7 @@ pub async fn run_server(
.nest("/inkeep", inkeep_oss::global_service())
.nest("/indexer", indexer_oss::management_service())
.nest("/mcp/w/{workspace_id}/list_tools", mcp_list_tools_service)
.nest("/db_health", db_health::global_service())
.nest("/health/detailed", health::detailed_service())
.nest(
"/saml",
@@ -21,6 +21,7 @@
import SettingsFooter from './workspaceSettings/SettingsFooter.svelte'
import SettingsPageHeader from './settings/SettingsPageHeader.svelte'
import WorkspaceRegistries from './instanceSettings/WorkspaceRegistries.svelte'
import DbHealth from './instanceSettings/DbHealth.svelte'
interface Props {
tab?: string
@@ -1052,6 +1053,12 @@
title="GitHub Enterprise App"
description="Configure a self-managed GitHub App for GitHub Enterprise Server git sync."
/>
{:else if category == 'DB Health'}
<SettingsPageHeader
title="DB Health"
description="On-demand database diagnostics. Analyze table sizes, job retention, connection pool health, vacuum status, and more."
/>
<DbHealth />
{:else if category == 'Auth/OAuth/SAML'}
<AuthSettings
bind:oauths
@@ -376,6 +376,7 @@ export const settings: Record<string, Setting[]> = {
}
],
'Auth/OAuth/SAML': [],
'DB Health': [],
Registries: [
{
label: 'Instance Python Version',
@@ -829,6 +830,12 @@ export const instanceSettingsNavigationGroups = [
aiId: 'instance-settings-indexer',
aiDescription: 'Instance indexer settings',
isEE: true
},
{
id: 'db_health',
label: 'DB Health',
aiId: 'instance-settings-db-health',
aiDescription: 'Database health diagnostics and performance insights'
}
]
},
@@ -900,7 +907,8 @@ export const tabToCategoryMap: Record<string, string> = {
jobs: 'Jobs',
private_hub: 'Private Hub',
github_enterprise_app: 'GitHub App',
websocket: 'WebSocket'
websocket: 'WebSocket',
db_health: 'DB Health'
}
export const tabToAuthSubTab: Record<string, 'sso' | 'oauth' | 'scim'> = {
@@ -933,7 +941,8 @@ export const categoryToTabMap: Record<string, string> = {
Jobs: 'jobs',
'Private Hub': 'private_hub',
'GitHub App': 'github_enterprise_app',
WebSocket: 'websocket'
WebSocket: 'websocket',
'DB Health': 'db_health'
}
export interface SearchableSettingItem {
@@ -0,0 +1,502 @@
<script lang="ts">
import { Button } from '$lib/components/common'
import { sendUserToast } from '$lib/toast'
import { Loader2, RefreshCw, ChevronDown, ChevronRight } from 'lucide-svelte'
let loading = $state(false)
let data: any = $state(null)
let error: string | null = $state(null)
let scanLimit = $state(10000)
const scanLimitOptions = [
{ label: '10,000', value: 10000 },
{ label: '50,000', value: 50000 },
{ label: '100,000', value: 100000 },
{ label: '500,000', value: 500000 }
]
let expandedSections: Record<string, boolean> = $state({
database_size: true,
job_retention: true,
large_results: true,
connection_pool: true,
table_maintenance: true,
slow_queries: true,
datatables: true
})
function toggleSection(key: string) {
expandedSections[key] = !expandedSections[key]
}
async function runDiagnostics() {
loading = true
error = null
try {
const response = await fetch(`/api/db_health?scan_limit=${scanLimit}`, {
credentials: 'include'
})
if (!response.ok) {
const text = await response.text()
throw new Error(text || `HTTP ${response.status}`)
}
data = await response.json()
} catch (e: any) {
error = e.message
sendUserToast('Failed to run diagnostics: ' + e.message, true)
} finally {
loading = false
}
}
function statusColor(status: string): string {
switch (status) {
case 'green':
return 'text-green-600 dark:text-green-400'
case 'yellow':
return 'text-yellow-600 dark:text-yellow-400'
case 'red':
return 'text-red-600 dark:text-red-400'
default:
return 'text-tertiary'
}
}
function statusBadge(status: string): string {
switch (status) {
case 'green':
return 'bg-green-100 text-green-800 dark:bg-green-900 dark:text-green-200'
case 'yellow':
return 'bg-yellow-100 text-yellow-800 dark:bg-yellow-900 dark:text-yellow-200'
case 'red':
return 'bg-red-100 text-red-800 dark:bg-red-900 dark:text-red-200'
default:
return 'bg-surface-secondary text-secondary'
}
}
function formatBytes(bytes: number): string {
if (bytes === 0) return '0 B'
const k = 1024
const sizes = ['B', 'KB', 'MB', 'GB', 'TB']
const i = Math.floor(Math.log(bytes) / Math.log(k))
return parseFloat((bytes / Math.pow(k, i)).toFixed(1)) + ' ' + sizes[i]
}
function formatNumber(n: number): string {
return n.toLocaleString()
}
function formatMs(ms: number): string {
if (ms < 1) return '<1 ms'
if (ms < 1000) return ms.toFixed(1) + ' ms'
return (ms / 1000).toFixed(2) + ' s'
}
function formatDate(d: string | null): string {
if (!d) return 'Never'
return new Date(d).toLocaleString()
}
</script>
<div class="flex flex-col gap-4">
<div class="flex items-center gap-2">
<Button
variant="default"
onclick={runDiagnostics}
disabled={loading}
startIcon={{ icon: loading ? Loader2 : RefreshCw }}
>
{loading ? 'Running...' : 'Run Diagnostics'}
</Button>
<label class="text-tertiary flex items-center gap-1 text-xs">
Scan last
<select
class="border-surface-secondary text-secondary rounded border bg-transparent px-1 py-0.5 text-xs"
bind:value={scanLimit}
>
{#each scanLimitOptions as opt}
<option value={opt.value}>{opt.label}</option>
{/each}
</select>
jobs
</label>
{#if loading}
<span class="text-tertiary text-xs">This may take a few seconds...</span>
{/if}
</div>
{#if error}
<div
class="rounded border border-red-300 bg-red-50 p-3 text-sm text-red-700 dark:border-red-700 dark:bg-red-950 dark:text-red-300"
>
{error}
</div>
{/if}
{#if data}
<!-- Database Size -->
<section class="border-surface-secondary rounded-md border">
<button
class="flex w-full items-center justify-between p-3 text-left hover:bg-surface-secondary/50"
onclick={() => toggleSection('database_size')}
>
<h3 class="text-primary text-sm font-semibold">Database Size</h3>
<div class="flex items-center gap-2">
<span class="text-tertiary text-xs">{data.database_size.total_size_pretty}</span>
{#if expandedSections.database_size}
<ChevronDown size={16} />
{:else}
<ChevronRight size={16} />
{/if}
</div>
</button>
{#if expandedSections.database_size}
<div class="border-surface-secondary border-t p-3">
<p class="text-secondary mb-2 text-xs">
Total database size: <strong>{data.database_size.total_size_pretty}</strong>
</p>
<div class="overflow-x-auto">
<table class="w-full text-left text-xs">
<thead>
<tr class="text-tertiary border-surface-secondary border-b">
<th class="pb-1 pr-4">Table</th>
<th class="pb-1 pr-4 text-right">Size</th>
</tr>
</thead>
<tbody>
{#each data.database_size.top_tables as t}
<tr class="border-surface-secondary border-b last:border-0">
<td class="text-primary py-1 pr-4 font-mono">{t.table_name}</td>
<td class="text-secondary py-1 pr-4 text-right">{t.total_size_pretty}</td>
</tr>
{/each}
</tbody>
</table>
</div>
</div>
{/if}
</section>
<!-- Job Retention -->
<section class="border-surface-secondary rounded-md border">
<button
class="flex w-full items-center justify-between p-3 text-left hover:bg-surface-secondary/50"
onclick={() => toggleSection('job_retention')}
>
<h3 class="text-primary text-sm font-semibold">Job Retention</h3>
<div class="flex items-center gap-2">
<span
class="rounded px-1.5 py-0.5 text-xs font-medium {statusBadge(
data.job_retention.status
)}"
>
{data.job_retention.status}
</span>
{#if expandedSections.job_retention}
<ChevronDown size={16} />
{:else}
<ChevronRight size={16} />
{/if}
</div>
</button>
{#if expandedSections.job_retention}
<div class="border-surface-secondary flex flex-col gap-1 border-t p-3 text-xs">
<p class="text-secondary">
Total completed jobs: <strong
>{formatNumber(data.job_retention.total_completed_jobs)}</strong
>
</p>
<p class="text-secondary">
Oldest job: <strong>{formatDate(data.job_retention.oldest_completed_at)}</strong>
</p>
<p class="text-secondary">
Retention period: <strong>
{data.job_retention.retention_period_secs
? formatNumber(data.job_retention.retention_period_secs) + 's'
: 'Not configured'}
</strong>
</p>
<p class="{statusColor(data.job_retention.status)} mt-1 font-medium">
{data.job_retention.message}
</p>
</div>
{/if}
</section>
<!-- Large Results -->
<section class="border-surface-secondary rounded-md border">
<button
class="flex w-full items-center justify-between p-3 text-left hover:bg-surface-secondary/50"
onclick={() => toggleSection('large_results')}
>
<h3 class="text-primary text-sm font-semibold"
>Large Job Results (last {scanLimit.toLocaleString()} jobs)</h3
>
<div class="flex items-center gap-2">
{#if data.large_results.avg_result_size_bytes != null}
<span class="text-tertiary text-xs"
>avg: {formatBytes(data.large_results.avg_result_size_bytes)}</span
>
{/if}
{#if expandedSections.large_results}
<ChevronDown size={16} />
{:else}
<ChevronRight size={16} />
{/if}
</div>
</button>
{#if expandedSections.large_results}
<div class="border-surface-secondary border-t p-3">
{#if data.large_results.top_large_results.length === 0}
<p class="text-tertiary text-xs">No job results found in the last 30 days.</p>
{:else}
<div class="overflow-x-auto">
<table class="w-full text-left text-xs">
<thead>
<tr class="text-tertiary border-surface-secondary border-b">
<th class="pb-1 pr-4">Job ID</th>
<th class="pb-1 pr-4">Workspace</th>
<th class="pb-1 pr-4">Script</th>
<th class="pb-1 pr-4 text-right">Result Size</th>
<th class="pb-1 pr-4 text-right">Completed</th>
</tr>
</thead>
<tbody>
{#each data.large_results.top_large_results as r}
<tr class="border-surface-secondary border-b last:border-0">
<td class="text-primary py-1 pr-4 font-mono">{r.id.substring(0, 8)}...</td>
<td class="text-secondary py-1 pr-4">{r.workspace_id}</td>
<td class="text-secondary py-1 pr-4 font-mono">{r.runnable_path ?? '-'}</td>
<td class="text-secondary py-1 pr-4 text-right"
>{formatBytes(r.result_size_bytes)}</td
>
<td class="text-secondary py-1 pr-4 text-right"
>{formatDate(r.completed_at)}</td
>
</tr>
{/each}
</tbody>
</table>
</div>
{/if}
</div>
{/if}
</section>
<!-- Connection Pool -->
<section class="border-surface-secondary rounded-md border">
<button
class="flex w-full items-center justify-between p-3 text-left hover:bg-surface-secondary/50"
onclick={() => toggleSection('connection_pool')}
>
<h3 class="text-primary text-sm font-semibold">Connection Pool</h3>
<div class="flex items-center gap-2">
<span
class="rounded px-1.5 py-0.5 text-xs font-medium {statusBadge(
data.connection_pool.status
)}"
>
{data.connection_pool.status}
</span>
{#if expandedSections.connection_pool}
<ChevronDown size={16} />
{:else}
<ChevronRight size={16} />
{/if}
</div>
</button>
{#if expandedSections.connection_pool}
<div class="border-surface-secondary flex flex-col gap-1 border-t p-3 text-xs">
<p class="text-secondary">
Pool size: <strong>{data.connection_pool.pool.size}</strong> / Max:
<strong>{data.connection_pool.pool.max_connections}</strong>
/ Idle: <strong>{data.connection_pool.pool.idle}</strong>
</p>
<p class="text-secondary">
Active PG connections: <strong>{data.connection_pool.pg_active_connections}</strong>
</p>
<p class="{statusColor(data.connection_pool.status)} mt-1 font-medium">
{data.connection_pool.message}
</p>
</div>
{/if}
</section>
<!-- Table Maintenance -->
<section class="border-surface-secondary rounded-md border">
<button
class="flex w-full items-center justify-between p-3 text-left hover:bg-surface-secondary/50"
onclick={() => toggleSection('table_maintenance')}
>
<h3 class="text-primary text-sm font-semibold">Table Maintenance (Vacuum/Bloat)</h3>
<div class="flex items-center gap-2">
{#if expandedSections.table_maintenance}
<ChevronDown size={16} />
{:else}
<ChevronRight size={16} />
{/if}
</div>
</button>
{#if expandedSections.table_maintenance}
<div class="border-surface-secondary border-t p-3">
<div class="overflow-x-auto">
<table class="w-full text-left text-xs">
<thead>
<tr class="text-tertiary border-surface-secondary border-b">
<th class="pb-1 pr-4">Table</th>
<th class="pb-1 pr-4 text-right">Live Tuples</th>
<th class="pb-1 pr-4 text-right">Dead Tuples</th>
<th class="pb-1 pr-4 text-right">Dead %</th>
<th class="pb-1 pr-4 text-right">Last Vacuum</th>
<th class="pb-1 pr-4 text-right">Last Analyze</th>
<th class="pb-1 pr-4">Status</th>
</tr>
</thead>
<tbody>
{#each data.table_maintenance as t}
<tr class="border-surface-secondary border-b last:border-0">
<td class="text-primary py-1 pr-4 font-mono">{t.table_name}</td>
<td class="text-secondary py-1 pr-4 text-right"
>{formatNumber(t.live_tuples)}</td
>
<td class="text-secondary py-1 pr-4 text-right"
>{formatNumber(t.dead_tuples)}</td
>
<td class="text-secondary py-1 pr-4 text-right"
>{(t.dead_ratio * 100).toFixed(1)}%</td
>
<td class="text-secondary py-1 pr-4 text-right"
>{formatDate(t.last_autovacuum)}</td
>
<td class="text-secondary py-1 pr-4 text-right"
>{formatDate(t.last_autoanalyze)}</td
>
<td class="py-1 pr-4">
<span
class="rounded px-1.5 py-0.5 text-xs font-medium {statusBadge(t.status)}"
>
{t.status}
</span>
</td>
</tr>
{/each}
</tbody>
</table>
</div>
</div>
{/if}
</section>
<!-- Slow Queries -->
<section class="border-surface-secondary rounded-md border">
<button
class="flex w-full items-center justify-between p-3 text-left hover:bg-surface-secondary/50"
onclick={() => toggleSection('slow_queries')}
>
<h3 class="text-primary text-sm font-semibold">Slow Queries</h3>
<div class="flex items-center gap-2">
{#if expandedSections.slow_queries}
<ChevronDown size={16} />
{:else}
<ChevronRight size={16} />
{/if}
</div>
</button>
{#if expandedSections.slow_queries}
<div class="border-surface-secondary border-t p-3">
{#if data.slow_queries == null}
<p class="text-tertiary text-xs">Slow query data not available.</p>
{:else if data.slow_queries.message}
<p class="text-tertiary text-xs">{data.slow_queries.message}</p>
{:else if data.slow_queries.queries.length === 0}
<p class="text-tertiary text-xs">No slow queries found.</p>
{:else}
<div class="overflow-x-auto">
<table class="w-full text-left text-xs">
<thead>
<tr class="text-tertiary border-surface-secondary border-b">
<th class="pb-1 pr-4">Query</th>
<th class="pb-1 pr-4 text-right">Calls</th>
<th class="pb-1 pr-4 text-right">Total Time</th>
<th class="pb-1 pr-4 text-right">Mean Time</th>
</tr>
</thead>
<tbody>
{#each data.slow_queries.queries as q}
<tr class="border-surface-secondary border-b last:border-0">
<td class="text-primary max-w-md truncate py-1 pr-4 font-mono">{q.query}</td>
<td class="text-secondary py-1 pr-4 text-right">{formatNumber(q.calls)}</td>
<td class="text-secondary py-1 pr-4 text-right"
>{formatMs(q.total_exec_time_ms)}</td
>
<td class="text-secondary py-1 pr-4 text-right"
>{formatMs(q.mean_exec_time_ms)}</td
>
</tr>
{/each}
</tbody>
</table>
</div>
{/if}
</div>
{/if}
</section>
<!-- Datatables -->
<section class="border-surface-secondary rounded-md border">
<button
class="flex w-full items-center justify-between p-3 text-left hover:bg-surface-secondary/50"
onclick={() => toggleSection('datatables')}
>
<h3 class="text-primary text-sm font-semibold">Datatables (Instance Storage)</h3>
<div class="flex items-center gap-2">
{#if expandedSections.datatables}
<ChevronDown size={16} />
{:else}
<ChevronRight size={16} />
{/if}
</div>
</button>
{#if expandedSections.datatables}
<div class="border-surface-secondary border-t p-3">
{#if data.datatables.length === 0}
<p class="text-tertiary text-xs">No instance-stored datatables found.</p>
{:else}
<div class="overflow-x-auto">
<table class="w-full text-left text-xs">
<thead>
<tr class="text-tertiary border-surface-secondary border-b">
<th class="pb-1 pr-4">Workspace</th>
<th class="pb-1 pr-4">Name</th>
<th class="pb-1 pr-4">Table</th>
<th class="pb-1 pr-4 text-right">Size</th>
<th class="pb-1 pr-4 text-right">Est. Rows</th>
</tr>
</thead>
<tbody>
{#each data.datatables as dt}
<tr class="border-surface-secondary border-b last:border-0">
<td class="text-secondary py-1 pr-4">{dt.workspace_id}</td>
<td class="text-primary py-1 pr-4">{dt.name}</td>
<td class="text-secondary py-1 pr-4 font-mono">{dt.table_name}</td>
<td class="text-secondary py-1 pr-4 text-right">{dt.size_pretty}</td>
<td class="text-secondary py-1 pr-4 text-right"
>{formatNumber(Math.round(dt.estimated_rows))}</td
>
</tr>
{/each}
</tbody>
</table>
</div>
{/if}
</div>
{/if}
</section>
{:else if !loading}
<p class="text-tertiary text-sm">
Click "Run Diagnostics" to analyze your database health. The queries are read-only and
lightweight.
</p>
{/if}
</div>