x

docs/how-to/how-to-profile-cpu.md

@@ -20,3 +20,31 @@ Sample at 49 Hertz, for 10 seconds, output report in text format.
 ```bash
 curl -X POST -s '0:4000/debug/prof/cpu?seconds=10&frequency=49&output=text' > /tmp/pprof.txt
 ```
+
+## Using `perf`
+
+First find the pid of GreptimeDB:
+
+Using `perf record` to profile GreptimeDB, at the sampling frequency of 99 hertz, and a duration of 60 seconds:
+
+```bash
+perf record -p <pid> --call-graph dwarf -F 99 -- sleep 60
+```
+
+The result will be saved to file `perf.data`.
+
+Then
+
+```bash
+perf script --no-inline > perf.out
+```
+
+Produce a flame graph out of it:
+
+```bash
+git clone https://github.com/brendangregg/FlameGraph
+
+FlameGraph/stackcollapse-perf.pl perf.out > perf.folded
+
+FlameGraph/flamegraph.pl perf.folded > perf.svg
+```

src/catalog/src/kvbackend/client.rs

@@ -18,7 +18,7 @@ use std::sync::atomic::{AtomicUsize, Ordering};
 use std::sync::{Arc, Mutex};
 use std::time::Duration;
 
-use common_error::ext::BoxedError;
+use common_error::ext::{BoxedError, ErrorExt};
 use common_meta::cache_invalidator::KvCacheInvalidator;
 use common_meta::error::Error::CacheNotGet;
 use common_meta::error::{CacheNotGetSnafu, Error, ExternalSnafu, GetKvCacheSnafu, Result};
@@ -37,6 +37,7 @@ use snafu::{OptionExt, ResultExt};
 
 use crate::metrics::{
     METRIC_CATALOG_KV_BATCH_GET, METRIC_CATALOG_KV_GET, METRIC_CATALOG_KV_REMOTE_GET,
+    METRIC_META_CLIENT_GET,
 };
 
 const DEFAULT_CACHE_MAX_CAPACITY: u64 = 10000;
@@ -292,7 +293,7 @@ impl KvBackend for CachedKvBackend {
         }
         .map_err(|e| {
             GetKvCacheSnafu {
-                err_msg: e.to_string(),
+                err_msg: e.output_msg(),
             }
             .build()
         });
@@ -445,6 +446,8 @@ impl KvBackend for MetaKvBackend {
     }
 
     async fn get(&self, key: &[u8]) -> Result<Option<KeyValue>> {
+        let _timer = METRIC_META_CLIENT_GET.start_timer();
+
         let mut response = self
             .client
             .range(RangeRequest::new().with_key(key))

src/catalog/src/metrics.rs

@@ -34,4 +34,6 @@ lazy_static! {
         register_histogram!("greptime_catalog_kv_get", "catalog kv get").unwrap();
     pub static ref METRIC_CATALOG_KV_BATCH_GET: Histogram =
         register_histogram!("greptime_catalog_kv_batch_get", "catalog kv batch get").unwrap();
+    pub static ref METRIC_META_CLIENT_GET: Histogram =
+        register_histogram!("greptime_meta_client_get", "meta client get").unwrap();
 }

src/cmd/src/datanode.rs

@@ -62,6 +62,13 @@ impl Instance {
     pub fn datanode(&self) -> &Datanode {
         &self.datanode
     }
+
+    /// Get mutable Datanode instance for changing some internal state, before starting it.
+    // Useful for wrapping Datanode instance. Please do not remove this method even if you find
+    // nowhere it is called.
+    pub fn datanode_mut(&mut self) -> &mut Datanode {
+        &mut self.datanode
+    }
 }
 
 #[async_trait]

src/common/meta/src/key/schema_name.rs

@@ -29,6 +29,7 @@ use crate::error::{self, Error, InvalidMetadataSnafu, ParseOptionSnafu, Result};
 use crate::key::{MetadataKey, SCHEMA_NAME_KEY_PATTERN, SCHEMA_NAME_KEY_PREFIX};
 use crate::kv_backend::txn::Txn;
 use crate::kv_backend::KvBackendRef;
+use crate::metrics::METRIC_META_SCHEMA_INFO_GET;
 use crate::range_stream::{PaginationStream, DEFAULT_PAGE_SIZE};
 use crate::rpc::store::RangeRequest;
 use crate::rpc::KeyValue;
@@ -209,6 +210,8 @@ impl SchemaManager {
         &self,
         schema: SchemaNameKey<'_>,
     ) -> Result<Option<DeserializedValueWithBytes<SchemaNameValue>>> {
+        let _timer = METRIC_META_SCHEMA_INFO_GET.start_timer();
+
         let raw_key = schema.to_bytes();
         self.kv_backend
             .get(&raw_key)

src/common/meta/src/key/table_info.rs

@@ -29,6 +29,7 @@ use crate::key::txn_helper::TxnOpGetResponseSet;
 use crate::key::{DeserializedValueWithBytes, MetadataKey, MetadataValue, TABLE_INFO_KEY_PREFIX};
 use crate::kv_backend::txn::Txn;
 use crate::kv_backend::KvBackendRef;
+use crate::metrics::METRIC_META_TABLE_INFO_GET;
 use crate::rpc::store::BatchGetRequest;
 
 /// The key stores the metadata of the table.
@@ -194,6 +195,8 @@ impl TableInfoManager {
         &self,
         table_id: TableId,
     ) -> Result<Option<DeserializedValueWithBytes<TableInfoValue>>> {
+        let _timer = METRIC_META_TABLE_INFO_GET.start_timer();
+
         let key = TableInfoKey::new(table_id);
         let raw_key = key.to_bytes();
         self.kv_backend

src/common/meta/src/metrics.rs

@@ -108,4 +108,9 @@ lazy_static! {
         &["name"]
     )
     .unwrap();
+
+    pub static ref METRIC_META_TABLE_INFO_GET: Histogram =
+        register_histogram!("greptime_meta_table_info_get", "get table info from kvbackend").unwrap();
+    pub static ref METRIC_META_SCHEMA_INFO_GET: Histogram =
+        register_histogram!("greptime_meta_schema_info_get", "get schema info from kvbackend").unwrap();
 }