diff --git a/Cargo.lock b/Cargo.lock index c52fa7acb9..cc37068d34 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8123,6 +8123,7 @@ version = "0.15.5" dependencies = [ "anyhow", "bytes", + "common-error", "common-telemetry", "common-test-util", "futures", diff --git a/src/common/error/src/lib.rs b/src/common/error/src/lib.rs index 0052d70cf3..18e6a0c9ae 100644 --- a/src/common/error/src/lib.rs +++ b/src/common/error/src/lib.rs @@ -45,3 +45,19 @@ pub fn from_err_code_msg_to_header(code: u32, msg: &str) -> HeaderMap { header.insert(GREPTIME_DB_HEADER_ERROR_MSG, msg); header } + +/// Returns the external root cause of the source error (exclude the current error). +pub fn root_source(err: &dyn std::error::Error) -> Option<&dyn std::error::Error> { + // There are some divergence about the behavior of the `sources()` API + // in https://github.com/rust-lang/rust/issues/58520 + // So this function iterates the sources manually. + let mut root = err.source(); + while let Some(r) = root { + if let Some(s) = r.source() { + root = Some(s); + } else { + break; + } + } + root +} diff --git a/src/object-store/Cargo.toml b/src/object-store/Cargo.toml index f90ea42d61..b76cee59f6 100644 --- a/src/object-store/Cargo.toml +++ b/src/object-store/Cargo.toml @@ -12,6 +12,7 @@ services-memory = ["opendal/services-memory"] [dependencies] bytes.workspace = true +common-error.workspace = true common-telemetry.workspace = true futures.workspace = true lazy_static.workspace = true diff --git a/src/object-store/src/util.rs b/src/object-store/src/util.rs index 271da33e85..e5e02c85d7 100644 --- a/src/object-store/src/util.rs +++ b/src/object-store/src/util.rs @@ -14,6 +14,7 @@ use std::fmt::Display; +use common_error::root_source; use common_telemetry::{debug, error, trace}; use opendal::layers::{LoggingInterceptor, LoggingLayer, TracingLayer}; use opendal::raw::{AccessorInfo, Operation}; @@ -159,11 +160,12 @@ impl LoggingInterceptor for DefaultLoggingInterceptor { err: Option<&opendal::Error>, ) { if let Some(err) = err { + let root = root_source(err); // Print error if it's unexpected, otherwise in error. if err.kind() == ErrorKind::Unexpected { error!( target: LOGGING_TARGET, - "service={} name={} {}: {operation} {message} {err:#?}", + "service={} name={} {}: {operation} {message} {err:#?}, root={root:#?}", info.scheme(), info.name(), LoggingContext(context), @@ -171,7 +173,7 @@ impl LoggingInterceptor for DefaultLoggingInterceptor { } else { debug!( target: LOGGING_TARGET, - "service={} name={} {}: {operation} {message} {err}", + "service={} name={} {}: {operation} {message} {err}, root={root:?}", info.scheme(), info.name(), LoggingContext(context),