From a8ddc72deedae97bdc3e1a7d170495e62bfe78bf Mon Sep 17 00:00:00 2001
From: Wez Furlong <wez@wezfurlong.org>
Date: Sun, 19 Feb 2023 12:44:15 -0700
Subject: [PATCH] add counters for transient and permanent failures

---
 crates/kumod/src/dest_site.rs      | 18 ++++++++++++++++++
 crates/kumod/src/metrics_helper.rs | 28 +++++++++++++++++++++++++++-
 2 files changed, 45 insertions(+), 1 deletion(-)

diff --git a/crates/kumod/src/dest_site.rs b/crates/kumod/src/dest_site.rs
index 4a69d369..3e49c4f2 100644
--- a/crates/kumod/src/dest_site.rs
+++ b/crates/kumod/src/dest_site.rs
@@ -190,6 +190,12 @@ impl SiteManager {
                 global_msgs_delivered: crate::metrics_helper::total_msgs_delivered_for_service(
                     "smtp_client",
                 ),
+                msgs_transfail: crate::metrics_helper::total_msgs_transfail_for_service(&service),
+                global_msgs_transfail: crate::metrics_helper::total_msgs_transfail_for_service(
+                    "smtp_client",
+                ),
+                msgs_fail: crate::metrics_helper::total_msgs_fail_for_service(&service),
+                global_msgs_fail: crate::metrics_helper::total_msgs_fail_for_service("smtp_client"),
             };
             let ready = Arc::new(StdMutex::new(HeapRb::new(site_config.max_ready)));
             let notify = Arc::new(Notify::new());
@@ -223,8 +229,15 @@ struct DeliveryMetrics {
     global_connection_gauge: IntGauge,
     connection_total: IntCounter,
     global_connection_total: IntCounter,
+
     msgs_delivered: IntCounter,
     global_msgs_delivered: IntCounter,
+
+    msgs_transfail: IntCounter,
+    global_msgs_transfail: IntCounter,
+
+    msgs_fail: IntCounter,
+    global_msgs_fail: IntCounter,
 }
 
 pub struct DestinationSite {
@@ -541,6 +554,9 @@ impl Dispatcher {
                 if let Some(msg) = self.msg.take() {
                     Self::requeue_message(msg, true).await?;
                 }
+                // FIXME: log transfail
+                self.metrics.msgs_transfail.inc();
+                self.metrics.global_msgs_transfail.inc();
                 tracing::debug!(
                     "failed to send message to {} {:?}: {response:?}",
                     self.name,
@@ -548,6 +564,8 @@ impl Dispatcher {
                 );
             }
             Err(ClientError::Rejected(response)) => {
+                self.metrics.msgs_fail.inc();
+                self.metrics.global_msgs_fail.inc();
                 tracing::error!(
                     "failed to send message to {} {:?}: {response:?}",
                     self.name,
diff --git a/crates/kumod/src/metrics_helper.rs b/crates/kumod/src/metrics_helper.rs
index c2f020db..a13dbedc 100644
--- a/crates/kumod/src/metrics_helper.rs
+++ b/crates/kumod/src/metrics_helper.rs
@@ -16,7 +16,19 @@ lazy_static::lazy_static! {
     pub static ref TOTAL_MSGS_DELIVERED: IntCounterVec = {
         prometheus::register_int_counter_vec!(
             "total_messages_delivered",
-            "total number messages ever delivered",
+            "total number of messages ever delivered",
+            &["service"]).unwrap()
+    };
+    pub static ref TOTAL_MSGS_TRANSFAIL: IntCounterVec = {
+        prometheus::register_int_counter_vec!(
+            "total_messages_transfail",
+            "total number of message delivery attempts that transiently failed",
+            &["service"]).unwrap()
+    };
+    pub static ref TOTAL_MSGS_FAIL: IntCounterVec = {
+        prometheus::register_int_counter_vec!(
+            "total_messages_fail",
+            "total number of message delivery attempts that permanently failed",
             &["service"]).unwrap()
     };
 }
@@ -35,6 +47,18 @@ pub fn total_msgs_delivered_for_service(service: &str) -> IntCounter {
         .unwrap()
 }
 
+pub fn total_msgs_transfail_for_service(service: &str) -> IntCounter {
+    TOTAL_MSGS_TRANSFAIL
+        .get_metric_with_label_values(&[service])
+        .unwrap()
+}
+
+pub fn total_msgs_fail_for_service(service: &str) -> IntCounter {
+    TOTAL_MSGS_FAIL
+        .get_metric_with_label_values(&[service])
+        .unwrap()
+}
+
 /// Remove metrics that are parameterized by a service name of
 /// some kind.
 ///
@@ -53,4 +77,6 @@ pub fn remove_metrics_for_service(service: &str) {
     CONN_GAUGE.remove_label_values(&[service]).ok();
     TOTAL_CONN.remove_label_values(&[service]).ok();
     TOTAL_MSGS_DELIVERED.remove_label_values(&[service]).ok();
+    TOTAL_MSGS_TRANSFAIL.remove_label_values(&[service]).ok();
+    TOTAL_MSGS_FAIL.remove_label_values(&[service]).ok();
 }