ready_queue: add deliver_message_latency_rollup

This tracks the duration of a deliver_message call by protocol.

I almost also added one of these per ready queue, but I'm unsure
of the overall memory requirements and whether it is 100% useful
to have that amount of granularity per ready queue, so I'm punting
for now.

crates/kumod/src/delivery_metrics.rs

@@ -1,4 +1,4 @@
-use prometheus::{IntCounter, IntGauge};
+use prometheus::{Histogram, IntCounter, IntGauge};
 
 #[derive(Clone, Debug)]
 pub struct DeliveryMetrics {
@@ -18,6 +18,8 @@ pub struct DeliveryMetrics {
 
     msgs_fail: IntCounter,
     global_msgs_fail: IntCounter,
+
+    pub deliver_message_rollup: Histogram,
 }
 
 impl DeliveryMetrics {
@@ -55,6 +57,7 @@ impl DeliveryMetrics {
             ),
             msgs_fail: crate::metrics_helper::total_msgs_fail_for_service(&service),
             global_msgs_fail: crate::metrics_helper::total_msgs_fail_for_service(service_type),
+            deliver_message_rollup: crate::metrics_helper::deliver_message_rollup_for_service(service_type),
         }
     }
 

crates/kumod/src/metrics_helper.rs

@@ -1,4 +1,4 @@
-use prometheus::{IntCounter, IntCounterVec, IntGauge, IntGaugeVec};
+use prometheus::{Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec};
 
 lazy_static::lazy_static! {
     pub static ref CONN_GAUGE: IntGaugeVec = {
@@ -55,6 +55,18 @@ lazy_static::lazy_static! {
             "number of times a message could not fit in the ready queue",
             &["service"]).unwrap()
     };
+    pub static ref DELIVER_MESSAGE_LATENCY_ROLLUP: HistogramVec = {
+        prometheus::register_histogram_vec!(
+            "deliver_message_latency_rollup",
+            "how long a deliver_message call takes for a given protocol",
+            &["service"]).unwrap()
+    };
+}
+
+pub fn deliver_message_rollup_for_service(service: &str) -> Histogram {
+    DELIVER_MESSAGE_LATENCY_ROLLUP
+        .get_metric_with_label_values(&[service])
+        .unwrap()
 }
 
 pub fn connection_denied_for_service(service: &str) -> IntCounter {

crates/kumod/src/ready_queue.rs

@@ -979,6 +979,8 @@ impl Dispatcher {
 
             connection_failures.clear();
             consecutive_connection_failures.store(0, Ordering::SeqCst);
+
+            let _timer_rollup = dispatcher.metrics.deliver_message_rollup.start_timer();
             dispatcher
                 .deliver_message(&mut *queue_dispatcher)
                 .await