diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 4fbe1e5c92..c2467509ee 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -465,6 +465,14 @@ pub async fn run_worker( ) .expect("register prometheus metric"); + let worker_execution_duration_counter = prometheus::register_int_counter!(prometheus::opts!( + "worker_execution_duration_counter", + "Total number of milliseconds spent executing jobs" + ) + .const_label("name", &worker_name)) + .expect("register prometheus metric"); + + let worker_sleep_duration = prometheus::register_histogram!(prometheus::HistogramOpts::new( "worker_sleep_duration", "Duration sleeping waiting for job", @@ -472,6 +480,15 @@ pub async fn run_worker( .const_label("name", &worker_name),) .expect("register prometheus metric"); + + let worker_sleep_duration_counter = prometheus::register_int_counter!(prometheus::opts!( + "worker_execution_sleep_counter", + "Total number of milliseconds spent sleeping between pulling jobs from the queue" + ) + .const_label("name", &worker_name)) + .expect("register prometheus metric"); + + let worker_pull_duration = prometheus::register_histogram!(prometheus::HistogramOpts::new( "worker_pull_duration", "Duration pulling next job", @@ -479,6 +496,13 @@ pub async fn run_worker( .const_label("name", &worker_name),) .expect("register prometheus metric"); + let worker_pull_duration_counter = prometheus::register_int_counter!(prometheus::opts!( + "worker_pull_sleep_counter", + "Total number of milliseconds spent pulling jobs (if growing large the db is undersized)" + ) + .const_label("name", &worker_name)) + .expect("register prometheus metric"); + let worker_execution_failed = prometheus::register_int_counter_vec!( prometheus::Opts::new("worker_execution_failed", "Number of failed jobs",) .const_label("name", &worker_name), @@ -573,7 +597,9 @@ pub async fn run_worker( }, (job, timer) = { let timer = worker_pull_duration.start_timer(); - pull(&db, WHITELIST_WORKSPACES.clone(), BLACKLIST_WORKSPACES.clone()).map(|x| (x, timer)) } => { + let instant = Instant::now(); + pull(&db, WHITELIST_WORKSPACES.clone(), BLACKLIST_WORKSPACES.clone()).map(|x| (x, timer))` } => { + worker_pull_duration_counter.inc_by((elapsed.as_millis() as i64).try_into().unwrap()); drop(timer); (false, job) },