diff --git a/docs/how-to/how-to-profile-memory.md b/docs/how-to/how-to-profile-memory.md index b4bc00093a7..da4e3b6f0a7 100644 --- a/docs/how-to/how-to-profile-memory.md +++ b/docs/how-to/how-to-profile-memory.md @@ -96,6 +96,21 @@ curl -X POST "localhost:4000/debug/prof/mem?output=proto" > greptime.pprof You can periodically dump profiling data and compare them to find the delta memory usage. +### Symbolicate external dump files + +If you have jemalloc heap dump files generated externally (e.g., via `MALLOC_CONF="prof:true,prof_prefix:jeprof.out,lg_prof_interval:26"` or `prof_gdump`), you can upload them to a running GreptimeDB instance for symbolication and flamegraph generation: + +```bash +# Upload a jemalloc heap dump file and get a symbolicated flamegraph +curl -X POST --data-binary @/path/to/jeprof.out.12345.0.i0.heap \ + localhost:4000/debug/prof/mem/symbol > flamegraph.svg +``` + +This is useful when: +- You collected heap dumps from a production environment +- You want to symbolicate dumps on a machine with debug symbols +- You need to analyze dumps generated by jemalloc's automatic dump mechanisms + ## Analyze profiling data with flamegraph To create flamegraph according to dumped profiling data: diff --git a/src/common/mem-prof/src/jemalloc.rs b/src/common/mem-prof/src/jemalloc.rs index a9359dad41e..e2164c3ac40 100644 --- a/src/common/mem-prof/src/jemalloc.rs +++ b/src/common/mem-prof/src/jemalloc.rs @@ -133,3 +133,23 @@ pub fn is_gdump_active() -> Result { // safety: PROF_GDUMP, if present, is a boolean value. unsafe { Ok(tikv_jemalloc_ctl::raw::read::(PROF_GDUMP).context(error::ReadGdumpSnafu)?) } } + +/// Symbolicate a jeheap format dump file and return a flamegraph. +/// +/// This function takes the raw content of a jemalloc heap dump file, +/// parses it using `parse_jeheap`, and generates a flamegraph SVG. +/// +/// The symbolication uses the current process's memory mappings. +pub fn symbolicate_jeheap(dump_content: &[u8]) -> Result> { + let profile = BufReader::new(dump_content); + let stack_profile = parse_jeheap(profile, MAPPINGS.as_deref()).context(ParseJeHeapSnafu)?; + + let mut opts = FlamegraphOptions::default(); + opts.title = "symbolicated_heap".to_string(); + opts.count_name = "bytes".to_string(); + let flamegraph = stack_profile + .to_flamegraph(&mut opts) + .context(FlamegraphSnafu)?; + + Ok(flamegraph) +} diff --git a/src/common/mem-prof/src/lib.rs b/src/common/mem-prof/src/lib.rs index 9ff67e72771..a570d1ee74f 100644 --- a/src/common/mem-prof/src/lib.rs +++ b/src/common/mem-prof/src/lib.rs @@ -19,7 +19,7 @@ mod jemalloc; #[cfg(not(windows))] pub use jemalloc::{ activate_heap_profile, deactivate_heap_profile, dump_flamegraph, dump_pprof, dump_profile, - is_gdump_active, is_heap_profile_active, set_gdump_active, + is_gdump_active, is_heap_profile_active, set_gdump_active, symbolicate_jeheap, }; #[cfg(windows)] @@ -61,3 +61,8 @@ pub fn is_gdump_active() -> error::Result { pub fn set_gdump_active(_: bool) -> error::Result<()> { error::ProfilingNotSupportedSnafu.fail() } + +#[cfg(windows)] +pub fn symbolicate_jeheap(_dump_content: &[u8]) -> error::Result> { + error::ProfilingNotSupportedSnafu.fail() +} diff --git a/src/servers/src/http.rs b/src/servers/src/http.rs index 29918249949..1745284692a 100644 --- a/src/servers/src/http.rs +++ b/src/servers/src/http.rs @@ -912,6 +912,7 @@ impl HttpServer { Router::new() .route("/cpu", routing::post(pprof::pprof_handler)) .route("/mem", routing::post(mem_prof::mem_prof_handler)) + .route("/mem/symbol", routing::post(mem_prof::symbolicate_handler)) .route( "/mem/activate", routing::post(mem_prof::activate_heap_prof_handler), diff --git a/src/servers/src/http/mem_prof.rs b/src/servers/src/http/mem_prof.rs index e6362aef3f5..c643fee8ddb 100644 --- a/src/servers/src/http/mem_prof.rs +++ b/src/servers/src/http/mem_prof.rs @@ -183,3 +183,26 @@ pub async fn gdump_status_handler() -> crate::error::Result { "The 'mem-prof' feature is disabled", )) } + +#[cfg(feature = "mem-prof")] +#[axum_macros::debug_handler] +pub async fn symbolicate_handler( + body: axum::body::Bytes, +) -> crate::error::Result { + use snafu::ResultExt; + + use crate::error::DumpProfileDataSnafu; + + let flamegraph = common_mem_prof::symbolicate_jeheap(&body).context(DumpProfileDataSnafu)?; + + Ok((StatusCode::OK, flamegraph)) +} + +#[cfg(not(feature = "mem-prof"))] +#[axum_macros::debug_handler] +pub async fn symbolicate_handler() -> crate::error::Result { + Ok(( + StatusCode::NOT_IMPLEMENTED, + "The 'mem-prof' feature is disabled", + )) +}