From 1e37847f4823f769f264eccc5d2aead0a9f224cc Mon Sep 17 00:00:00 2001 From: luofucong Date: Wed, 27 Nov 2024 17:23:19 +0800 Subject: [PATCH] x --- docs/how-to/how-to-profile-cpu.md | 28 ++++++++++++++++++++++++++ src/catalog/src/kvbackend/client.rs | 7 +++++-- src/catalog/src/metrics.rs | 2 ++ src/cmd/src/datanode.rs | 7 +++++++ src/common/meta/src/key/schema_name.rs | 3 +++ src/common/meta/src/key/table_info.rs | 3 +++ src/common/meta/src/metrics.rs | 5 +++++ 7 files changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/how-to/how-to-profile-cpu.md b/docs/how-to/how-to-profile-cpu.md index ff50b0ea44..9bc43cbf6e 100644 --- a/docs/how-to/how-to-profile-cpu.md +++ b/docs/how-to/how-to-profile-cpu.md @@ -20,3 +20,31 @@ Sample at 49 Hertz, for 10 seconds, output report in text format. ```bash curl -X POST -s '0:4000/debug/prof/cpu?seconds=10&frequency=49&output=text' > /tmp/pprof.txt ``` + +## Using `perf` + +First find the pid of GreptimeDB: + +Using `perf record` to profile GreptimeDB, at the sampling frequency of 99 hertz, and a duration of 60 seconds: + +```bash +perf record -p --call-graph dwarf -F 99 -- sleep 60 +``` + +The result will be saved to file `perf.data`. + +Then + +```bash +perf script --no-inline > perf.out +``` + +Produce a flame graph out of it: + +```bash +git clone https://github.com/brendangregg/FlameGraph + +FlameGraph/stackcollapse-perf.pl perf.out > perf.folded + +FlameGraph/flamegraph.pl perf.folded > perf.svg +``` diff --git a/src/catalog/src/kvbackend/client.rs b/src/catalog/src/kvbackend/client.rs index 7a09861850..c4b4024f1b 100644 --- a/src/catalog/src/kvbackend/client.rs +++ b/src/catalog/src/kvbackend/client.rs @@ -18,7 +18,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use std::time::Duration; -use common_error::ext::BoxedError; +use common_error::ext::{BoxedError, ErrorExt}; use common_meta::cache_invalidator::KvCacheInvalidator; use common_meta::error::Error::CacheNotGet; use common_meta::error::{CacheNotGetSnafu, Error, ExternalSnafu, GetKvCacheSnafu, Result}; @@ -37,6 +37,7 @@ use snafu::{OptionExt, ResultExt}; use crate::metrics::{ METRIC_CATALOG_KV_BATCH_GET, METRIC_CATALOG_KV_GET, METRIC_CATALOG_KV_REMOTE_GET, + METRIC_META_CLIENT_GET, }; const DEFAULT_CACHE_MAX_CAPACITY: u64 = 10000; @@ -292,7 +293,7 @@ impl KvBackend for CachedKvBackend { } .map_err(|e| { GetKvCacheSnafu { - err_msg: e.to_string(), + err_msg: e.output_msg(), } .build() }); @@ -445,6 +446,8 @@ impl KvBackend for MetaKvBackend { } async fn get(&self, key: &[u8]) -> Result> { + let _timer = METRIC_META_CLIENT_GET.start_timer(); + let mut response = self .client .range(RangeRequest::new().with_key(key)) diff --git a/src/catalog/src/metrics.rs b/src/catalog/src/metrics.rs index 77063d64ba..323bd4c5f4 100644 --- a/src/catalog/src/metrics.rs +++ b/src/catalog/src/metrics.rs @@ -34,4 +34,6 @@ lazy_static! { register_histogram!("greptime_catalog_kv_get", "catalog kv get").unwrap(); pub static ref METRIC_CATALOG_KV_BATCH_GET: Histogram = register_histogram!("greptime_catalog_kv_batch_get", "catalog kv batch get").unwrap(); + pub static ref METRIC_META_CLIENT_GET: Histogram = + register_histogram!("greptime_meta_client_get", "meta client get").unwrap(); } diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs index be2aedf57e..0364fa76eb 100644 --- a/src/cmd/src/datanode.rs +++ b/src/cmd/src/datanode.rs @@ -62,6 +62,13 @@ impl Instance { pub fn datanode(&self) -> &Datanode { &self.datanode } + + /// Get mutable Datanode instance for changing some internal state, before starting it. + // Useful for wrapping Datanode instance. Please do not remove this method even if you find + // nowhere it is called. + pub fn datanode_mut(&mut self) -> &mut Datanode { + &mut self.datanode + } } #[async_trait] diff --git a/src/common/meta/src/key/schema_name.rs b/src/common/meta/src/key/schema_name.rs index 41d4f503b1..f3e3295909 100644 --- a/src/common/meta/src/key/schema_name.rs +++ b/src/common/meta/src/key/schema_name.rs @@ -29,6 +29,7 @@ use crate::error::{self, Error, InvalidMetadataSnafu, ParseOptionSnafu, Result}; use crate::key::{MetadataKey, SCHEMA_NAME_KEY_PATTERN, SCHEMA_NAME_KEY_PREFIX}; use crate::kv_backend::txn::Txn; use crate::kv_backend::KvBackendRef; +use crate::metrics::METRIC_META_SCHEMA_INFO_GET; use crate::range_stream::{PaginationStream, DEFAULT_PAGE_SIZE}; use crate::rpc::store::RangeRequest; use crate::rpc::KeyValue; @@ -209,6 +210,8 @@ impl SchemaManager { &self, schema: SchemaNameKey<'_>, ) -> Result>> { + let _timer = METRIC_META_SCHEMA_INFO_GET.start_timer(); + let raw_key = schema.to_bytes(); self.kv_backend .get(&raw_key) diff --git a/src/common/meta/src/key/table_info.rs b/src/common/meta/src/key/table_info.rs index 3ea40d4f1c..bc5e6398db 100644 --- a/src/common/meta/src/key/table_info.rs +++ b/src/common/meta/src/key/table_info.rs @@ -29,6 +29,7 @@ use crate::key::txn_helper::TxnOpGetResponseSet; use crate::key::{DeserializedValueWithBytes, MetadataKey, MetadataValue, TABLE_INFO_KEY_PREFIX}; use crate::kv_backend::txn::Txn; use crate::kv_backend::KvBackendRef; +use crate::metrics::METRIC_META_TABLE_INFO_GET; use crate::rpc::store::BatchGetRequest; /// The key stores the metadata of the table. @@ -194,6 +195,8 @@ impl TableInfoManager { &self, table_id: TableId, ) -> Result>> { + let _timer = METRIC_META_TABLE_INFO_GET.start_timer(); + let key = TableInfoKey::new(table_id); let raw_key = key.to_bytes(); self.kv_backend diff --git a/src/common/meta/src/metrics.rs b/src/common/meta/src/metrics.rs index 4626074f73..dfa6e3cc87 100644 --- a/src/common/meta/src/metrics.rs +++ b/src/common/meta/src/metrics.rs @@ -108,4 +108,9 @@ lazy_static! { &["name"] ) .unwrap(); + + pub static ref METRIC_META_TABLE_INFO_GET: Histogram = + register_histogram!("greptime_meta_table_info_get", "get table info from kvbackend").unwrap(); + pub static ref METRIC_META_SCHEMA_INFO_GET: Histogram = + register_histogram!("greptime_meta_schema_info_get", "get schema info from kvbackend").unwrap(); }