add counters for transient and permanent failures

This commit is contained in:
Wez Furlong
2023-02-19 12:45:04 -07:00
parent 5a8280626b
commit a8ddc72dee
2 changed files with 45 additions and 1 deletions
+18
View File
@@ -190,6 +190,12 @@ impl SiteManager {
global_msgs_delivered: crate::metrics_helper::total_msgs_delivered_for_service(
"smtp_client",
),
msgs_transfail: crate::metrics_helper::total_msgs_transfail_for_service(&service),
global_msgs_transfail: crate::metrics_helper::total_msgs_transfail_for_service(
"smtp_client",
),
msgs_fail: crate::metrics_helper::total_msgs_fail_for_service(&service),
global_msgs_fail: crate::metrics_helper::total_msgs_fail_for_service("smtp_client"),
};
let ready = Arc::new(StdMutex::new(HeapRb::new(site_config.max_ready)));
let notify = Arc::new(Notify::new());
@@ -223,8 +229,15 @@ struct DeliveryMetrics {
global_connection_gauge: IntGauge,
connection_total: IntCounter,
global_connection_total: IntCounter,
msgs_delivered: IntCounter,
global_msgs_delivered: IntCounter,
msgs_transfail: IntCounter,
global_msgs_transfail: IntCounter,
msgs_fail: IntCounter,
global_msgs_fail: IntCounter,
}
pub struct DestinationSite {
@@ -541,6 +554,9 @@ impl Dispatcher {
if let Some(msg) = self.msg.take() {
Self::requeue_message(msg, true).await?;
}
// FIXME: log transfail
self.metrics.msgs_transfail.inc();
self.metrics.global_msgs_transfail.inc();
tracing::debug!(
"failed to send message to {} {:?}: {response:?}",
self.name,
@@ -548,6 +564,8 @@ impl Dispatcher {
);
}
Err(ClientError::Rejected(response)) => {
self.metrics.msgs_fail.inc();
self.metrics.global_msgs_fail.inc();
tracing::error!(
"failed to send message to {} {:?}: {response:?}",
self.name,
+27 -1
View File
@@ -16,7 +16,19 @@ lazy_static::lazy_static! {
pub static ref TOTAL_MSGS_DELIVERED: IntCounterVec = {
prometheus::register_int_counter_vec!(
"total_messages_delivered",
"total number messages ever delivered",
"total number of messages ever delivered",
&["service"]).unwrap()
};
pub static ref TOTAL_MSGS_TRANSFAIL: IntCounterVec = {
prometheus::register_int_counter_vec!(
"total_messages_transfail",
"total number of message delivery attempts that transiently failed",
&["service"]).unwrap()
};
pub static ref TOTAL_MSGS_FAIL: IntCounterVec = {
prometheus::register_int_counter_vec!(
"total_messages_fail",
"total number of message delivery attempts that permanently failed",
&["service"]).unwrap()
};
}
@@ -35,6 +47,18 @@ pub fn total_msgs_delivered_for_service(service: &str) -> IntCounter {
.unwrap()
}
pub fn total_msgs_transfail_for_service(service: &str) -> IntCounter {
TOTAL_MSGS_TRANSFAIL
.get_metric_with_label_values(&[service])
.unwrap()
}
pub fn total_msgs_fail_for_service(service: &str) -> IntCounter {
TOTAL_MSGS_FAIL
.get_metric_with_label_values(&[service])
.unwrap()
}
/// Remove metrics that are parameterized by a service name of
/// some kind.
///
@@ -53,4 +77,6 @@ pub fn remove_metrics_for_service(service: &str) {
CONN_GAUGE.remove_label_values(&[service]).ok();
TOTAL_CONN.remove_label_values(&[service]).ok();
TOTAL_MSGS_DELIVERED.remove_label_values(&[service]).ok();
TOTAL_MSGS_TRANSFAIL.remove_label_values(&[service]).ok();
TOTAL_MSGS_FAIL.remove_label_values(&[service]).ok();
}