feat: adds metrics, tracing and retry layer to object-store (#621)

Cargo.lock

@@ -1789,6 +1789,7 @@ dependencies = [
  "axum 0.6.0-rc.2",
  "axum-macros",
  "axum-test-helper",
+ "backon",
  "catalog",
  "client",
  "common-base",
@@ -3665,6 +3666,7 @@ dependencies = [
  "http",
  "log",
  "md-5",
+ "metrics",
  "once_cell",
  "parking_lot",
  "percent-encoding",
@@ -3677,6 +3679,7 @@ dependencies = [
  "thiserror",
  "time 0.3.14",
  "tokio",
+ "tracing",
  "ureq",
 ]
 

src/datanode/Cargo.toml

@@ -13,6 +13,7 @@ api = { path = "../api" }
 async-trait = "0.1"
 axum = "0.6.0-rc.2"
 axum-macros = "0.3.0-rc.1"
+backon = "0.2"
 catalog = { path = "../catalog" }
 common-base = { path = "../common/base" }
 common-catalog = { path = "../common/catalog" }

src/datanode/src/instance.rs

@@ -16,6 +16,7 @@ use std::sync::Arc;
 use std::time::Duration;
 use std::{fs, path};
 
+use backon::ExponentialBackoff;
 use catalog::remote::MetaKvBackend;
 use catalog::CatalogManagerRef;
 use common_grpc::channel_manager::{ChannelConfig, ChannelManager};
@@ -26,7 +27,7 @@ use meta_client::client::{MetaClient, MetaClientBuilder};
 use meta_client::MetaClientOpts;
 use mito::config::EngineConfig as TableEngineConfig;
 use mito::engine::MitoEngine;
-use object_store::layers::LoggingLayer;
+use object_store::layers::{LoggingLayer, MetricsLayer, RetryLayer, TracingLayer};
 use object_store::services::fs::Builder;
 use object_store::{util, ObjectStore};
 use query::query_engine::{QueryEngineFactory, QueryEngineRef};
@@ -189,7 +190,15 @@ pub(crate) async fn new_object_store(store_config: &ObjectStoreConfig) -> Result
         .build()
         .context(error::InitBackendSnafu { dir: &data_dir })?;
 
-    let object_store = ObjectStore::new(accessor).layer(LoggingLayer); // Add logging
+    let object_store = ObjectStore::new(accessor)
+        // Add retry
+        .layer(RetryLayer::new(ExponentialBackoff::default().with_jitter()))
+        // Add metrics
+        .layer(MetricsLayer)
+        // Add logging
+        .layer(LoggingLayer)
+        // Add tracing
+        .layer(TracingLayer);
 
     Ok(object_store)
 }

src/object-store/Cargo.toml

@@ -6,7 +6,7 @@ license = "Apache-2.0"
 
 [dependencies]
 futures = { version = "0.3" }
-opendal = "0.20"
+opendal = { version = "0.20", features = ["layers-tracing", "layers-metrics"]}
 tokio = { version = "1.0", features = ["full"] }
 
 [dev-dependencies]