chore: add more metrics about parquet and cache (#4410)

* chore: add more metrics about parquet and cache

* resolve PR comments

* resolve PR comments

* resolve PR comments

* resolve PR comments

src/mito2/src/cache.rs

@@ -27,6 +27,7 @@ use std::sync::Arc;
 
 use datatypes::value::Value;
 use datatypes::vectors::VectorRef;
+use moka::notification::RemovalCause;
 use moka::sync::Cache;
 use parquet::column::page::Page;
 use parquet::file::metadata::ParquetMetaData;
@@ -36,7 +37,7 @@ use crate::cache::cache_size::parquet_meta_size;
 use crate::cache::file_cache::{FileType, IndexKey};
 use crate::cache::index::{InvertedIndexCache, InvertedIndexCacheRef};
 use crate::cache::write_cache::WriteCacheRef;
-use crate::metrics::{CACHE_BYTES, CACHE_HIT, CACHE_MISS};
+use crate::metrics::{CACHE_BYTES, CACHE_EVICTION, CACHE_HIT, CACHE_MISS};
 use crate::read::Batch;
 use crate::sst::file::FileId;
 
@@ -273,15 +274,27 @@ impl CacheManagerBuilder {
 
     /// Builds the [CacheManager].
     pub fn build(self) -> CacheManager {
+        fn to_str(cause: RemovalCause) -> &'static str {
+            match cause {
+                RemovalCause::Expired => "expired",
+                RemovalCause::Explicit => "explicit",
+                RemovalCause::Replaced => "replaced",
+                RemovalCause::Size => "size",
+            }
+        }
+
         let sst_meta_cache = (self.sst_meta_cache_size != 0).then(|| {
             Cache::builder()
                 .max_capacity(self.sst_meta_cache_size)
                 .weigher(meta_cache_weight)
-                .eviction_listener(|k, v, _cause| {
+                .eviction_listener(|k, v, cause| {
                     let size = meta_cache_weight(&k, &v);
                     CACHE_BYTES
                         .with_label_values(&[SST_META_TYPE])
                         .sub(size.into());
+                    CACHE_EVICTION
+                        .with_label_values(&[SST_META_TYPE, to_str(cause)])
+                        .inc();
                 })
                 .build()
         });
@@ -289,11 +302,14 @@ impl CacheManagerBuilder {
             Cache::builder()
                 .max_capacity(self.vector_cache_size)
                 .weigher(vector_cache_weight)
-                .eviction_listener(|k, v, _cause| {
+                .eviction_listener(|k, v, cause| {
                     let size = vector_cache_weight(&k, &v);
                     CACHE_BYTES
                         .with_label_values(&[VECTOR_TYPE])
                         .sub(size.into());
+                    CACHE_EVICTION
+                        .with_label_values(&[VECTOR_TYPE, to_str(cause)])
+                        .inc();
                 })
                 .build()
         });
@@ -301,9 +317,12 @@ impl CacheManagerBuilder {
             Cache::builder()
                 .max_capacity(self.page_cache_size)
                 .weigher(page_cache_weight)
-                .eviction_listener(|k, v, _cause| {
+                .eviction_listener(|k, v, cause| {
                     let size = page_cache_weight(&k, &v);
                     CACHE_BYTES.with_label_values(&[PAGE_TYPE]).sub(size.into());
+                    CACHE_EVICTION
+                        .with_label_values(&[PAGE_TYPE, to_str(cause)])
+                        .inc();
                 })
                 .build()
         });
@@ -313,11 +332,14 @@ impl CacheManagerBuilder {
             Cache::builder()
                 .max_capacity(self.selector_result_cache_size)
                 .weigher(selector_result_cache_weight)
-                .eviction_listener(|k, v, _cause| {
+                .eviction_listener(|k, v, cause| {
                     let size = selector_result_cache_weight(&k, &v);
                     CACHE_BYTES
                         .with_label_values(&[SELECTOR_RESULT_TYPE])
                         .sub(size.into());
+                    CACHE_EVICTION
+                        .with_label_values(&[SELECTOR_RESULT_TYPE, to_str(cause)])
+                        .inc();
                 })
                 .build()
         });

src/mito2/src/metrics.rs

@@ -19,6 +19,7 @@ use prometheus::*;
 pub const STAGE_LABEL: &str = "stage";
 /// Type label.
 pub const TYPE_LABEL: &str = "type";
+const CACHE_EVICTION_CAUSE: &str = "cause";
 /// Reason to flush.
 pub const FLUSH_REASON: &str = "reason";
 /// File type label.
@@ -190,6 +191,12 @@ lazy_static! {
         "mito upload bytes total",
     )
     .unwrap();
+    /// Cache eviction counter, labeled with cache type and eviction reason.
+    pub static ref CACHE_EVICTION: IntCounterVec = register_int_counter_vec!(
+        "greptime_mito_cache_eviction",
+        "mito cache eviction",
+        &[TYPE_LABEL, CACHE_EVICTION_CAUSE]
+    ).unwrap();
     // ------- End of cache metrics.
 
     // Index metrics.

src/mito2/src/sst/parquet/reader.rs

@@ -302,6 +302,10 @@ impl ParquetReaderBuilder {
         file_path: &str,
         file_size: u64,
     ) -> Result<Arc<ParquetMetaData>> {
+        let _t = READ_STAGE_ELAPSED
+            .with_label_values(&["read_parquet_metadata"])
+            .start_timer();
+
         let region_id = self.file_handle.region_id();
         let file_id = self.file_handle.file_id();
         // Tries to get from global cache.