From 91cdbc150da94918dbd06b057c99f0117179eee4 Mon Sep 17 00:00:00 2001
From: Wez Furlong <wez@wezfurlong.org>
Date: Fri, 23 Aug 2024 17:15:57 -0700
Subject: [PATCH] ready_queue: add deliver_message_latency_rollup

This tracks the duration of a deliver_message call by protocol.

I almost also added one of these per ready queue, but I'm unsure
of the overall memory requirements and whether it is 100% useful
to have that amount of granularity per ready queue, so I'm punting
for now.
---
 crates/kumod/src/delivery_metrics.rs |  5 ++++-
 crates/kumod/src/metrics_helper.rs   | 14 +++++++++++++-
 crates/kumod/src/ready_queue.rs      |  2 ++
 3 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/crates/kumod/src/delivery_metrics.rs b/crates/kumod/src/delivery_metrics.rs
index bc6577da..9fd614fb 100644
--- a/crates/kumod/src/delivery_metrics.rs
+++ b/crates/kumod/src/delivery_metrics.rs
@@ -1,4 +1,4 @@
-use prometheus::{IntCounter, IntGauge};
+use prometheus::{Histogram, IntCounter, IntGauge};
 
 #[derive(Clone, Debug)]
 pub struct DeliveryMetrics {
@@ -18,6 +18,8 @@ pub struct DeliveryMetrics {
 
     msgs_fail: IntCounter,
     global_msgs_fail: IntCounter,
+
+    pub deliver_message_rollup: Histogram,
 }
 
 impl DeliveryMetrics {
@@ -55,6 +57,7 @@ impl DeliveryMetrics {
             ),
             msgs_fail: crate::metrics_helper::total_msgs_fail_for_service(&service),
             global_msgs_fail: crate::metrics_helper::total_msgs_fail_for_service(service_type),
+            deliver_message_rollup: crate::metrics_helper::deliver_message_rollup_for_service(service_type),
         }
     }
 
diff --git a/crates/kumod/src/metrics_helper.rs b/crates/kumod/src/metrics_helper.rs
index 4eccd594..23027109 100644
--- a/crates/kumod/src/metrics_helper.rs
+++ b/crates/kumod/src/metrics_helper.rs
@@ -1,4 +1,4 @@
-use prometheus::{IntCounter, IntCounterVec, IntGauge, IntGaugeVec};
+use prometheus::{Histogram, HistogramVec, IntCounter, IntCounterVec, IntGauge, IntGaugeVec};
 
 lazy_static::lazy_static! {
     pub static ref CONN_GAUGE: IntGaugeVec = {
@@ -55,6 +55,18 @@ lazy_static::lazy_static! {
             "number of times a message could not fit in the ready queue",
             &["service"]).unwrap()
     };
+    pub static ref DELIVER_MESSAGE_LATENCY_ROLLUP: HistogramVec = {
+        prometheus::register_histogram_vec!(
+            "deliver_message_latency_rollup",
+            "how long a deliver_message call takes for a given protocol",
+            &["service"]).unwrap()
+    };
+}
+
+pub fn deliver_message_rollup_for_service(service: &str) -> Histogram {
+    DELIVER_MESSAGE_LATENCY_ROLLUP
+        .get_metric_with_label_values(&[service])
+        .unwrap()
 }
 
 pub fn connection_denied_for_service(service: &str) -> IntCounter {
diff --git a/crates/kumod/src/ready_queue.rs b/crates/kumod/src/ready_queue.rs
index ec7d5038..ed67035c 100644
--- a/crates/kumod/src/ready_queue.rs
+++ b/crates/kumod/src/ready_queue.rs
@@ -979,6 +979,8 @@ impl Dispatcher {
 
             connection_failures.clear();
             consecutive_connection_failures.store(0, Ordering::SeqCst);
+
+            let _timer_rollup = dispatcher.metrics.deliver_message_rollup.start_timer();
             dispatcher
                 .deliver_message(&mut *queue_dispatcher)
                 .await