diff --git a/crates/kumod/src/delivery_metrics.rs b/crates/kumod/src/delivery_metrics.rs index 9fd614fb..30d554a0 100644 --- a/crates/kumod/src/delivery_metrics.rs +++ b/crates/kumod/src/delivery_metrics.rs @@ -57,7 +57,9 @@ impl DeliveryMetrics { ), msgs_fail: crate::metrics_helper::total_msgs_fail_for_service(&service), global_msgs_fail: crate::metrics_helper::total_msgs_fail_for_service(service_type), - deliver_message_rollup: crate::metrics_helper::deliver_message_rollup_for_service(service_type), + deliver_message_rollup: crate::metrics_helper::deliver_message_rollup_for_service( + service_type, + ), } } diff --git a/crates/kumod/src/metrics_helper.rs b/crates/kumod/src/metrics_helper.rs index 23027109..7518a407 100644 --- a/crates/kumod/src/metrics_helper.rs +++ b/crates/kumod/src/metrics_helper.rs @@ -61,6 +61,12 @@ lazy_static::lazy_static! { "how long a deliver_message call takes for a given protocol", &["service"]).unwrap() }; + pub static ref TOTAL_READYQ_RUNS: IntCounter = { + prometheus::register_int_counter!( + "total_readyq_runs", + "total number of times a readyq maintainer was run" + ).unwrap() + }; } pub fn deliver_message_rollup_for_service(service: &str) -> Histogram { diff --git a/crates/kumod/src/queue.rs b/crates/kumod/src/queue.rs index ad3e18b0..f2ab4293 100644 --- a/crates/kumod/src/queue.rs +++ b/crates/kumod/src/queue.rs @@ -68,6 +68,12 @@ lazy_static::lazy_static! { "number of times a message was delayed due throttle_insert_ready_queue event", &["queue"]).unwrap() }; + static ref TOTAL_QMAINT_RUNS: IntCounter = { + prometheus::register_int_counter!( + "total_qmaint_runs", + "total number of times a scheduled queue maintainer was run" + ).unwrap() + }; pub static ref QMAINT_RUNTIME: Runtime = Runtime::new( "qmaint", |cpus| cpus/4, &QMAINT_THREADS).unwrap(); @@ -1592,6 +1598,8 @@ async fn maintain_named_queue(q: &QueueHandle) -> anyhow::Result<()> { _ = tokio::time::sleep_until(reap_at.into()), if queue_is_empty => {"reap"} }; + TOTAL_QMAINT_RUNS.inc(); + { tracing::debug!( "maintaining {} {:?} which has {} entries. wakeup after {:?} reason={reason}", diff --git a/crates/kumod/src/ready_queue.rs b/crates/kumod/src/ready_queue.rs index ed67035c..4b2ac5c3 100644 --- a/crates/kumod/src/ready_queue.rs +++ b/crates/kumod/src/ready_queue.rs @@ -6,6 +6,7 @@ use crate::http_server::admin_suspend_ready_q_v1::{ }; use crate::logging::disposition::{log_disposition, LogDisposition, RecordType}; use crate::lua_deliver::LuaQueueDispatcher; +use crate::metrics_helper::TOTAL_READYQ_RUNS; use crate::queue::{DeliveryProto, Queue, QueueConfig, QueueManager, QMAINT_RUNTIME}; use crate::smtp_dispatcher::{MxListEntry, OpportunisticInsecureTlsHandshakeError, SmtpDispatcher}; use crate::spool::SpoolManager; @@ -384,6 +385,8 @@ impl ReadyQueueManager { }, }; + TOTAL_READYQ_RUNS.inc(); + if last_config_refresh.elapsed() >= ONE_MINUTE && !queue.activity.is_shutting_down() { last_config_refresh = tokio::time::Instant::now(); match Self::compute_config(