chore: print root cause in opendal logging interceptor (#7183) (#7557)

* chore: print root cause in opendal



* refactor: extract a function root_source() to get the cause



---------

Signed-off-by: evenyag <realevenyag@gmail.com>

Cargo.lock

@@ -8123,6 +8123,7 @@ version = "0.15.5"
 dependencies = [
  "anyhow",
  "bytes",
+ "common-error",
  "common-telemetry",
  "common-test-util",
  "futures",

src/common/error/src/lib.rs

@@ -45,3 +45,19 @@ pub fn from_err_code_msg_to_header(code: u32, msg: &str) -> HeaderMap {
     header.insert(GREPTIME_DB_HEADER_ERROR_MSG, msg);
     header
 }
+
+/// Returns the external root cause of the source error (exclude the current error).
+pub fn root_source(err: &dyn std::error::Error) -> Option<&dyn std::error::Error> {
+    // There are some divergence about the behavior of the `sources()` API
+    // in https://github.com/rust-lang/rust/issues/58520
+    // So this function iterates the sources manually.
+    let mut root = err.source();
+    while let Some(r) = root {
+        if let Some(s) = r.source() {
+            root = Some(s);
+        } else {
+            break;
+        }
+    }
+    root
+}

src/object-store/Cargo.toml

@@ -12,6 +12,7 @@ services-memory = ["opendal/services-memory"]
 
 [dependencies]
 bytes.workspace = true
+common-error.workspace = true
 common-telemetry.workspace = true
 futures.workspace = true
 lazy_static.workspace = true

src/object-store/src/util.rs

@@ -14,6 +14,7 @@
 
 use std::fmt::Display;
 
+use common_error::root_source;
 use common_telemetry::{debug, error, trace};
 use opendal::layers::{LoggingInterceptor, LoggingLayer, TracingLayer};
 use opendal::raw::{AccessorInfo, Operation};
@@ -159,11 +160,12 @@ impl LoggingInterceptor for DefaultLoggingInterceptor {
         err: Option<&opendal::Error>,
     ) {
         if let Some(err) = err {
+            let root = root_source(err);
             // Print error if it's unexpected, otherwise in error.
             if err.kind() == ErrorKind::Unexpected {
                 error!(
                     target: LOGGING_TARGET,
-                    "service={} name={} {}: {operation} {message} {err:#?}",
+                    "service={} name={} {}: {operation} {message} {err:#?}, root={root:#?}",
                     info.scheme(),
                     info.name(),
                     LoggingContext(context),
@@ -171,7 +173,7 @@ impl LoggingInterceptor for DefaultLoggingInterceptor {
             } else {
                 debug!(
                     target: LOGGING_TARGET,
-                    "service={} name={} {}: {operation} {message} {err}",
+                    "service={} name={} {}: {operation} {message} {err}, root={root:?}",
                     info.scheme(),
                     info.name(),
                     LoggingContext(context),