feat: add region_statistics table (#4771)

* refactor: introduce `region_statistic`

* refactor: move DatanodeStat related structs to common_meta

* chore: add comments

* feat: implement `list_region_stats` for `ClusterInfo` trait

* feat: add `region_statistics` table

* feat: add table_id and region_number fields

* chore: rename unused snafu

* chore: udpate sqlness results

* chore: avoid to print source in error msg

* chore: move `procedure_info` under `greptime` catalog

* chore: apply suggestions from CR

* Update src/common/meta/src/datanode.rs

Co-authored-by: jeremyhi <jiachun_feng@proton.me>

---------

Co-authored-by: jeremyhi <jiachun_feng@proton.me>

src/mito2/src/engine.rs

@@ -76,7 +76,8 @@ use store_api::logstore::provider::Provider;
 use store_api::logstore::LogStore;
 use store_api::metadata::RegionMetadataRef;
 use store_api::region_engine::{
-    BatchResponses, RegionEngine, RegionRole, RegionScannerRef, SetReadonlyResponse,
+    BatchResponses, RegionEngine, RegionRole, RegionScannerRef, RegionStatistic,
+    SetReadonlyResponse,
 };
 use store_api::region_request::{AffectedRows, RegionOpenRequest, RegionRequest};
 use store_api::storage::{RegionId, ScanRequest};
@@ -89,7 +90,6 @@ use crate::error::{
 use crate::manifest::action::RegionEdit;
 use crate::metrics::HANDLE_REQUEST_ELAPSED;
 use crate::read::scan_region::{ScanParallism, ScanRegion, Scanner};
-use crate::region::RegionUsage;
 use crate::request::{RegionEditRequest, WorkerRequest};
 use crate::wal::entry_distributor::{
     build_wal_entry_distributor_and_receivers, DEFAULT_ENTRY_RECEIVER_BUFFER_SIZE,
@@ -133,15 +133,12 @@ impl MitoEngine {
         self.inner.workers.is_region_opening(region_id)
     }
 
-    /// Returns the region disk/memory usage information.
-    pub fn get_region_usage(&self, region_id: RegionId) -> Result<RegionUsage> {
-        let region = self
-            .inner
+    /// Returns the region disk/memory statistic.
+    pub fn get_region_statistic(&self, region_id: RegionId) -> Option<RegionStatistic> {
+        self.inner
             .workers
             .get_region(region_id)
-            .context(RegionNotFoundSnafu { region_id })?;
-
-        Ok(region.region_usage())
+            .map(|region| region.region_statistic())
     }
 
     /// Handle substrait query and return a stream of record batches
@@ -546,12 +543,8 @@ impl RegionEngine for MitoEngine {
         self.inner.stop().await.map_err(BoxedError::new)
     }
 
-    fn region_disk_usage(&self, region_id: RegionId) -> Option<i64> {
-        let size = self
-            .get_region_usage(region_id)
-            .map(|usage| usage.disk_usage())
-            .ok()?;
-        size.try_into().ok()
+    fn region_statistic(&self, region_id: RegionId) -> Option<RegionStatistic> {
+        self.get_region_statistic(region_id)
     }
 
     fn set_writable(&self, region_id: RegionId, writable: bool) -> Result<(), BoxedError> {

src/mito2/src/engine/basic_test.rs

@@ -552,8 +552,8 @@ async fn test_region_usage() {
         .unwrap();
     // region is empty now, check manifest size
     let region = engine.get_region(region_id).unwrap();
-    let region_stat = region.region_usage();
-    assert_eq!(region_stat.manifest_usage, 686);
+    let region_stat = region.region_statistic();
+    assert_eq!(region_stat.manifest_size, 686);
 
     // put some rows
     let rows = Rows {
@@ -563,8 +563,8 @@ async fn test_region_usage() {
 
     put_rows(&engine, region_id, rows).await;
 
-    let region_stat = region.region_usage();
-    assert!(region_stat.wal_usage > 0);
+    let region_stat = region.region_statistic();
+    assert!(region_stat.wal_size > 0);
 
     // delete some rows
     let rows = Rows {
@@ -573,18 +573,18 @@ async fn test_region_usage() {
     };
     delete_rows(&engine, region_id, rows).await;
 
-    let region_stat = region.region_usage();
-    assert!(region_stat.wal_usage > 0);
+    let region_stat = region.region_statistic();
+    assert!(region_stat.wal_size > 0);
 
     // flush region
     flush_region(&engine, region_id, None).await;
 
-    let region_stat = region.region_usage();
-    assert_eq!(region_stat.sst_usage, 3010);
+    let region_stat = region.region_statistic();
+    assert_eq!(region_stat.sst_size, 3010);
 
     // region total usage
     // Some memtables may share items.
-    assert!(region_stat.disk_usage() >= 4028);
+    assert!(region_stat.estimated_disk_size() >= 4028);
 }
 
 #[tokio::test]

src/mito2/src/region.rs

@@ -28,6 +28,7 @@ use crossbeam_utils::atomic::AtomicCell;
 use snafu::{ensure, OptionExt};
 use store_api::logstore::provider::Provider;
 use store_api::metadata::RegionMetadataRef;
+use store_api::region_engine::RegionStatistic;
 use store_api::storage::RegionId;
 
 use crate::access_layer::AccessLayerRef;
@@ -252,10 +253,8 @@ impl MitoRegion {
         }
     }
 
-    /// Returns the region usage in bytes.
-    pub(crate) fn region_usage(&self) -> RegionUsage {
-        let region_id = self.region_id;
-
+    /// Returns the region statistic.
+    pub(crate) fn region_statistic(&self) -> RegionStatistic {
         let version = self.version();
         let memtables = &version.memtables;
         let memtable_usage = (memtables.mutable_usage() + memtables.immutables_usage()) as u64;
@@ -265,11 +264,11 @@ impl MitoRegion {
         let wal_usage = self.estimated_wal_usage(memtable_usage);
         let manifest_usage = self.stats.total_manifest_size();
 
-        RegionUsage {
-            region_id,
-            wal_usage,
-            sst_usage,
-            manifest_usage,
+        RegionStatistic {
+            memtable_size: memtable_usage,
+            wal_size: wal_usage,
+            manifest_size: manifest_usage,
+            sst_size: sst_usage,
         }
     }