diff --git a/backend/windmill-api/src/db_health.rs b/backend/windmill-api/src/db_health.rs index 5b881331c0..412c126dec 100644 --- a/backend/windmill-api/src/db_health.rs +++ b/backend/windmill-api/src/db_health.rs @@ -6,7 +6,12 @@ * LICENSE-AGPL for a copy of the license. */ -use axum::{extract::Query, routing::get, Extension, Json, Router}; +use axum::{ + extract::Query, + http::StatusCode, + routing::{get, post}, + Extension, Json, Router, +}; use serde::{Deserialize, Serialize}; use windmill_common::error::JsonResult; @@ -15,7 +20,11 @@ use crate::db::{ApiAuthed, DB}; use crate::utils::require_super_admin; pub fn global_service() -> Router { - Router::new().route("/", get(get_db_health)) + Router::new() + .route("/", get(get_db_health)) + .route("/jobs", get(get_db_health_jobs)) + .route("/slow_queries", get(get_slow_queries)) + .route("/slow_queries/reset", post(reset_slow_queries)) } // --- Response types --- @@ -31,14 +40,18 @@ pub enum HealthLevel { #[derive(Serialize)] pub struct DbHealthResponse { pub database_size: DatabaseSizeInfo, - pub job_retention: JobRetentionInfo, - pub large_results: LargeResultsInfo, pub connection_pool: ConnectionPoolInfo, pub table_maintenance: Vec, pub slow_queries: Option, pub datatables: Vec, } +#[derive(Serialize)] +pub struct DbHealthJobsResponse { + pub job_retention: JobRetentionInfo, + pub large_results: LargeResultsInfo, +} + #[derive(Serialize)] pub struct DatabaseSizeInfo { pub total_size_bytes: i64, @@ -102,6 +115,8 @@ pub struct TableMaintenanceInfo { pub struct SlowQueriesInfo { pub queries: Vec, pub message: Option, + /// When stats were last reset (from pg_stat_statements_info.stats_reset, PG 14+) + pub stats_reset: Option>, } #[derive(Serialize)] @@ -130,37 +145,45 @@ struct DbHealthQuery { scan_limit: Option, } +#[derive(Deserialize, Clone, Copy)] +#[serde(rename_all = "snake_case")] +enum SlowQuerySort { + Total, + Mean, + Calls, +} + +impl SlowQuerySort { + fn order_by(&self) -> &'static str { + match self { + SlowQuerySort::Total => "total_exec_time DESC", + SlowQuerySort::Mean => "mean_exec_time DESC", + SlowQuerySort::Calls => "calls DESC", + } + } +} + +#[derive(Deserialize)] +struct SlowQueriesQuery { + sort: Option, +} + async fn get_db_health( ApiAuthed { email, .. }: ApiAuthed, Extension(db): Extension, - Query(query): Query, ) -> JsonResult { require_super_admin(&db, &email).await?; - let scan_limit = query.scan_limit.unwrap_or(10_000).clamp(1_000, 1_000_000); - - let ( - database_size, - job_retention, - large_results, - connection_pool, - table_maintenance, - slow_queries, - datatables, - ) = tokio::try_join!( + let (database_size, connection_pool, table_maintenance, slow_queries, datatables) = tokio::try_join!( fetch_database_size(&db), - fetch_job_retention(&db), - fetch_large_results(&db, scan_limit), fetch_connection_pool(&db), fetch_table_maintenance(&db), - fetch_slow_queries(&db), + fetch_slow_queries(&db, SlowQuerySort::Total), fetch_datatables(&db), )?; Ok(Json(DbHealthResponse { database_size, - job_retention, - large_results, connection_pool, table_maintenance, slow_queries, @@ -168,6 +191,49 @@ async fn get_db_health( })) } +async fn get_db_health_jobs( + ApiAuthed { email, .. }: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult { + require_super_admin(&db, &email).await?; + + let scan_limit = query.scan_limit.unwrap_or(10_000).clamp(1_000, 1_000_000); + + let (job_retention, large_results) = tokio::try_join!( + fetch_job_retention(&db), + fetch_large_results(&db, scan_limit), + )?; + + Ok(Json(DbHealthJobsResponse { job_retention, large_results })) +} + +async fn get_slow_queries( + ApiAuthed { email, .. }: ApiAuthed, + Extension(db): Extension, + Query(query): Query, +) -> JsonResult> { + require_super_admin(&db, &email).await?; + let sort = query.sort.unwrap_or(SlowQuerySort::Total); + Ok(Json(fetch_slow_queries(&db, sort).await?)) +} + +async fn reset_slow_queries( + ApiAuthed { email, .. }: ApiAuthed, + Extension(db): Extension, +) -> windmill_common::error::Result { + require_super_admin(&db, &email).await?; + sqlx::query("SELECT pg_stat_statements_reset()") + .execute(&db) + .await + .map_err(|e| { + windmill_common::error::Error::InternalErr(format!( + "Failed to reset pg_stat_statements (is the extension enabled?): {e}" + )) + })?; + Ok(StatusCode::NO_CONTENT) +} + // --- Diagnostic queries --- async fn fetch_database_size(db: &DB) -> windmill_common::error::Result { @@ -407,6 +473,7 @@ async fn fetch_table_maintenance( LEFT JOIN pg_class p ON p.oid = i.inhparent ) sub GROUP BY table_name + HAVING SUM(live_tuples) + SUM(dead_tuples) >= 1000 ORDER BY SUM(dead_tuples) DESC"# ) .fetch_all(db) @@ -441,7 +508,10 @@ async fn fetch_table_maintenance( .collect()) } -async fn fetch_slow_queries(db: &DB) -> windmill_common::error::Result> { +async fn fetch_slow_queries( + db: &DB, + sort: SlowQuerySort, +) -> windmill_common::error::Result> { let ext_exists: bool = sqlx::query_scalar!( r#"SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'pg_stat_statements') as "exists!""# ) @@ -455,35 +525,54 @@ async fn fetch_slow_queries(db: &DB) -> windmill_common::error::Result = sqlx::query_as::<_, (String, i64, f64, f64)>( + // Use raw query since pg_stat_statements may not exist at compile time. + // ORDER BY column is controlled via an enum (SlowQuerySort) so only safe + // whitelisted column names reach the query — no SQL injection risk. + let query = format!( r#"SELECT - LEFT(query, 200), + LEFT(query, 500), calls::bigint, total_exec_time::float8, mean_exec_time::float8 FROM pg_stat_statements WHERE query NOT LIKE '%pg_stat_statements%' - ORDER BY mean_exec_time DESC - LIMIT 10"#, - ) - .fetch_all(db) - .await? - .into_iter() - .map( - |(query, calls, total_exec_time_ms, mean_exec_time_ms)| SlowQueryRow { - query, - calls, - total_exec_time_ms, - mean_exec_time_ms, - }, - ) - .collect(); + ORDER BY {} + LIMIT 50"#, + sort.order_by() + ); + let rows: Vec = sqlx::query_as::<_, (String, i64, f64, f64)>(&query) + .fetch_all(db) + .await? + .into_iter() + .map( + |(query, calls, total_exec_time_ms, mean_exec_time_ms)| SlowQueryRow { + query, + calls, + total_exec_time_ms, + mean_exec_time_ms, + }, + ) + .collect(); - Ok(Some(SlowQueriesInfo { queries: rows, message: None })) + // pg_stat_statements_info exists in PG 14+; tolerate its absence + let stats_reset: Option> = + sqlx::query_scalar::<_, Option>>( + "SELECT stats_reset FROM pg_stat_statements_info", + ) + .fetch_one(db) + .await + .ok() + .flatten(); + + Ok(Some(SlowQueriesInfo { + queries: rows, + message: None, + stats_reset, + })) } async fn fetch_datatables(db: &DB) -> windmill_common::error::Result> { diff --git a/frontend/src/lib/components/instanceSettings/DbHealth.svelte b/frontend/src/lib/components/instanceSettings/DbHealth.svelte index 9614422552..b1a16bdad5 100644 --- a/frontend/src/lib/components/instanceSettings/DbHealth.svelte +++ b/frontend/src/lib/components/instanceSettings/DbHealth.svelte @@ -1,12 +1,75 @@
-
- - - {#if loading} - This may take a few seconds... - {/if} -
+ + + + - {#if error} -
- {error} + {#if activeTab === 'overview'} +
+
+ + {#if error} +
+ {error} +
+ {/if} + + {#if data} + +
+ + {#if expandedSections.database_size} +
+

+ Total database size: {data.database_size.total_size_pretty} +

+
+ + + + + + + + + {#each data.database_size.top_tables as t} + + + + + {/each} + +
TableSize
{t.table_name}{t.total_size_pretty}
+
+
+ {/if} +
+ + +
+ + {#if expandedSections.connection_pool} +
+

+ Total connections: {data.connection_pool.pg_total_connections} / Max: + {data.connection_pool.pg_max_connections} +

+

+ Active: {data.connection_pool.pg_active_connections} + / Idle: {data.connection_pool.pg_idle_connections} +

+

+ {data.connection_pool.message} +

+
+ {/if} +
+ + +
+ + {#if expandedSections.table_maintenance} +
+
+ + + + + + + + + + + + + + {#each data.table_maintenance as t} + + + + + + + + + + {/each} + +
TableLive TuplesDead TuplesDead %Last VacuumLast AnalyzeStatus
{t.table_name}{formatNumber(t.live_tuples)}{formatNumber(t.dead_tuples)}{(t.dead_ratio * 100).toFixed(1)}%{formatDate(t.last_autovacuum)}{formatDate(t.last_autoanalyze)} + + {t.status} + +
+
+
+ {/if} +
+ + +
+ + {#if expandedSections.slow_queries} +
+ {#if data.slow_queries == null} +

Slow query data not available.

+ {:else if data.slow_queries.message} +
+

{data.slow_queries.message}

+
+

+ Setup (requires superuser + a postgres restart): +

+
    +
  1. + On the postgres server, enable the preload library: +
    ALTER SYSTEM SET shared_preload_libraries = 'pg_stat_statements';
    +
  2. +
  3. + Restart postgres so the library is actually loaded (e.g. sudo systemctl restart postgresql, or use your managed DB console for RDS/Cloud SQL). +
  4. +
  5. + On this database, create the extension: +
    CREATE EXTENSION pg_stat_statements;
    +
  6. +
  7. Click Refresh on this page.
  8. +
+
+
+ {:else} +
+
+
+ Top 50 queries from pg_stat_statements, sorted server-side. Click a column to + re-sort. Click a row to show the full query. + {#if slowSortLoading} + + {/if} +
+ {#if data.slow_queries.stats_reset} + Stats since: {formatDate(data.slow_queries.stats_reset)} + {/if} +
+
+ +
+
+ {#if data.slow_queries.queries.length === 0} +

No slow queries found.

+ {:else} +
+ + + + + + + + + + + {#each data.slow_queries.queries as q, i} + (expandedQueries[i] = !expandedQueries[i])} + > + + + + + + {/each} + +
Query + + + + + +
+ {#if expandedQueries[i]} +
{q.query}
+ {:else} +
{q.query}
+ {/if} +
{formatNumber(q.calls)}{formatMs(q.total_exec_time_ms)}{formatMs(q.mean_exec_time_ms)}
+
+ {/if} + {/if} +
+ {/if} +
+ + +
+ + {#if expandedSections.datatables} +
+ {#if data.datatables.length === 0} +

No instance-stored datatables found.

+ {:else} +
+ + + + + + + + + + + + {#each data.datatables as dt} + + + + + + + + {/each} + +
WorkspaceNameTableSizeEst. Rows
{dt.workspace_id}{dt.name}{dt.table_name}{dt.size_pretty}{formatNumber(Math.round(dt.estimated_rows))}
+
+ {/if} +
+ {/if} +
+ {/if} {/if} - {#if data} - -
- - {#if expandedSections.database_size} -
-

- Total database size: {data.database_size.total_size_pretty} -

-
- - - - - - - - - {#each data.database_size.top_tables as t} - - - - - {/each} - -
TableSize
{t.table_name}{t.total_size_pretty}
-
-
- {/if} -
+ {jobsLoading ? 'Scanning...' : 'Refresh'} + +
+ {#if jobsError} +
+ {jobsError} +
+ {/if}
{#if expandedSections.job_retention} -
-

- Total completed jobs: {formatNumber(data.job_retention.total_completed_jobs)} -

-

- Oldest job: {formatDate(data.job_retention.oldest_completed_at)} -

-

- Retention period: - {data.job_retention.retention_period_secs - ? formatNumber(data.job_retention.retention_period_secs) + 's' - : 'Not configured'} - -

-

- {data.job_retention.message} -

+
+ {#if jobsData} +
+

+ Total completed jobs: {formatNumber(jobsData.job_retention.total_completed_jobs)} +

+

+ Oldest job: {formatDate(jobsData.job_retention.oldest_completed_at)} +

+

+ Retention period: + {jobsData.job_retention.retention_period_secs + ? formatNumber(jobsData.job_retention.retention_period_secs) + 's' + : 'Not configured'} + +

+

+ {jobsData.job_retention.message} +

+
+ {:else} +

Click Refresh to load.

+ {/if}
{/if}
@@ -234,10 +667,12 @@ >Large Job Results (last {scanLimit.toLocaleString()} jobs)
- {#if data.large_results.avg_result_size_bytes != null} + {#if jobsData && jobsData.large_results.avg_result_size_bytes != null} avg: {formatBytes(data.large_results.avg_result_size_bytes)}avg: {formatBytes(jobsData.large_results.avg_result_size_bytes)} + {:else if jobsLoading} + {/if} {#if expandedSections.large_results} @@ -248,7 +683,9 @@ {#if expandedSections.large_results}
- {#if data.large_results.top_large_results.length === 0} + {#if !jobsData} +

Click Refresh to load.

+ {:else if jobsData.large_results.top_large_results.length === 0}

No job results larger than 1 KB found in the scanned jobs.

@@ -265,7 +702,7 @@ - {#each data.large_results.top_large_results as r} + {#each jobsData.large_results.top_large_results as r} {/if} - - -
- - {#if expandedSections.connection_pool} -
-

- Total connections: {data.connection_pool.pg_total_connections} / Max: - {data.connection_pool.pg_max_connections} -

-

- Active: {data.connection_pool.pg_active_connections} - / Idle: {data.connection_pool.pg_idle_connections} -

-

- {data.connection_pool.message} -

-
- {/if} -
- - -
- - {#if expandedSections.table_maintenance} -
-
- - - - - - - - - - - - - - {#each data.table_maintenance as t} - - - - - - - - - - {/each} - -
TableLive TuplesDead TuplesDead %Last VacuumLast AnalyzeStatus
{t.table_name}{formatNumber(t.live_tuples)}{formatNumber(t.dead_tuples)}{(t.dead_ratio * 100).toFixed(1)}%{formatDate(t.last_autovacuum)}{formatDate(t.last_autoanalyze)} - - {t.status} - -
-
-
- {/if} -
- - -
- - {#if expandedSections.slow_queries} -
- {#if data.slow_queries == null} -

Slow query data not available.

- {:else if data.slow_queries.message} -

{data.slow_queries.message}

- {:else if data.slow_queries.queries.length === 0} -

No slow queries found.

- {:else} -
- - - - - - - - - - - {#each data.slow_queries.queries as q} - - - - - - - {/each} - -
QueryCallsTotal TimeMean Time
{q.query}{formatNumber(q.calls)}{formatMs(q.total_exec_time_ms)}{formatMs(q.mean_exec_time_ms)}
-
- {/if} -
- {/if} -
- - -
- - {#if expandedSections.datatables} -
- {#if data.datatables.length === 0} -

No instance-stored datatables found.

- {:else} -
- - - - - - - - - - - - {#each data.datatables as dt} - - - - - - - - {/each} - -
WorkspaceNameTableSizeEst. Rows
{dt.workspace_id}{dt.name}{dt.table_name}{dt.size_pretty}{formatNumber(Math.round(dt.estimated_rows))}
-
- {/if} -
- {/if} -
- {:else if !loading} -

- Click "Run Diagnostics" to analyze your database health. The queries are read-only and - lightweight. -

{/if}