From d3860671a8e48512f432c0d1c8638bcaaf461cea Mon Sep 17 00:00:00 2001
From: Weny Xu <wenymedia@gmail.com>
Date: Tue, 28 May 2024 17:05:37 +0900
Subject: [PATCH] chore: add `LAST_SENT_HEARTBEAT_ELAPSED` metric (#4062)

---
 src/datanode/src/heartbeat.rs | 5 +++++
 src/datanode/src/metrics.rs   | 6 ++++++
 2 files changed, 11 insertions(+)

diff --git a/src/datanode/src/heartbeat.rs b/src/datanode/src/heartbeat.rs
index b1cdd61859..ed518fcebe 100644
--- a/src/datanode/src/heartbeat.rs
+++ b/src/datanode/src/heartbeat.rs
@@ -213,6 +213,7 @@ impl HeartbeatTask {
         let epoch = self.region_alive_keeper.epoch();
 
         self.region_alive_keeper.start(Some(event_receiver)).await?;
+        let mut last_sent = Instant::now();
 
         common_runtime::spawn_bg(async move {
             let sleep = tokio::time::sleep(Duration::from_millis(0));
@@ -271,6 +272,10 @@ impl HeartbeatTask {
                     }
                 };
                 if let Some(req) = req {
+                    metrics::LAST_SENT_HEARTBEAT_ELAPSED
+                        .set(last_sent.elapsed().as_millis() as i64);
+                    // Resets the timer.
+                    last_sent = Instant::now();
                     debug!("Sending heartbeat request: {:?}", req);
                     if let Err(e) = tx.send(req).await {
                         error!(e; "Failed to send heartbeat to metasrv");
diff --git a/src/datanode/src/metrics.rs b/src/datanode/src/metrics.rs
index 7eba418695..1c660a315c 100644
--- a/src/datanode/src/metrics.rs
+++ b/src/datanode/src/metrics.rs
@@ -35,6 +35,12 @@ lazy_static! {
         "last received heartbeat lease elapsed",
     )
     .unwrap();
+    /// The elapsed time since the last sent heartbeat.
+    pub static ref LAST_SENT_HEARTBEAT_ELAPSED: IntGauge = register_int_gauge!(
+        "greptime_last_sent_heartbeat_lease_elapsed",
+        "last sent heartbeat lease elapsed",
+    )
+    .unwrap();
     pub static ref LEASE_EXPIRED_REGION: IntGaugeVec = register_int_gauge_vec!(
         "greptime_lease_expired_region",
         "lease expired region",