queue/ready_queue: add counter for number of maintainer runs

crates/kumod/src/delivery_metrics.rs

@@ -57,7 +57,9 @@ impl DeliveryMetrics {
             ),
             msgs_fail: crate::metrics_helper::total_msgs_fail_for_service(&service),
             global_msgs_fail: crate::metrics_helper::total_msgs_fail_for_service(service_type),
-            deliver_message_rollup: crate::metrics_helper::deliver_message_rollup_for_service(service_type),
+            deliver_message_rollup: crate::metrics_helper::deliver_message_rollup_for_service(
+                service_type,
+            ),
         }
     }
 

crates/kumod/src/metrics_helper.rs

@@ -61,6 +61,12 @@ lazy_static::lazy_static! {
             "how long a deliver_message call takes for a given protocol",
             &["service"]).unwrap()
     };
+    pub static ref TOTAL_READYQ_RUNS: IntCounter = {
+        prometheus::register_int_counter!(
+            "total_readyq_runs",
+            "total number of times a readyq maintainer was run"
+            ).unwrap()
+    };
 }
 
 pub fn deliver_message_rollup_for_service(service: &str) -> Histogram {

crates/kumod/src/queue.rs

@@ -68,6 +68,12 @@ lazy_static::lazy_static! {
             "number of times a message was delayed due throttle_insert_ready_queue event",
             &["queue"]).unwrap()
     };
+    static ref TOTAL_QMAINT_RUNS: IntCounter = {
+        prometheus::register_int_counter!(
+            "total_qmaint_runs",
+            "total number of times a scheduled queue maintainer was run"
+            ).unwrap()
+    };
 
     pub static ref QMAINT_RUNTIME: Runtime = Runtime::new(
         "qmaint", |cpus| cpus/4, &QMAINT_THREADS).unwrap();
@@ -1592,6 +1598,8 @@ async fn maintain_named_queue(q: &QueueHandle) -> anyhow::Result<()> {
             _ = tokio::time::sleep_until(reap_at.into()), if queue_is_empty => {"reap"}
         };
 
+        TOTAL_QMAINT_RUNS.inc();
+
         {
             tracing::debug!(
                 "maintaining {} {:?} which has {} entries. wakeup after {:?} reason={reason}",

crates/kumod/src/ready_queue.rs

@@ -6,6 +6,7 @@ use crate::http_server::admin_suspend_ready_q_v1::{
 };
 use crate::logging::disposition::{log_disposition, LogDisposition, RecordType};
 use crate::lua_deliver::LuaQueueDispatcher;
+use crate::metrics_helper::TOTAL_READYQ_RUNS;
 use crate::queue::{DeliveryProto, Queue, QueueConfig, QueueManager, QMAINT_RUNTIME};
 use crate::smtp_dispatcher::{MxListEntry, OpportunisticInsecureTlsHandshakeError, SmtpDispatcher};
 use crate::spool::SpoolManager;
@@ -384,6 +385,8 @@ impl ReadyQueueManager {
                 },
             };
 
+            TOTAL_READYQ_RUNS.inc();
+
             if last_config_refresh.elapsed() >= ONE_MINUTE && !queue.activity.is_shutting_down() {
                 last_config_refresh = tokio::time::Instant::now();
                 match Self::compute_config(