diff --git a/Cargo.lock b/Cargo.lock index 21c57b60ed..7b2f738a97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1789,6 +1789,7 @@ dependencies = [ "axum 0.6.0-rc.2", "axum-macros", "axum-test-helper", + "backon", "catalog", "client", "common-base", @@ -3665,6 +3666,7 @@ dependencies = [ "http", "log", "md-5", + "metrics", "once_cell", "parking_lot", "percent-encoding", @@ -3677,6 +3679,7 @@ dependencies = [ "thiserror", "time 0.3.14", "tokio", + "tracing", "ureq", ] diff --git a/src/datanode/Cargo.toml b/src/datanode/Cargo.toml index c673947a47..f96ec55977 100644 --- a/src/datanode/Cargo.toml +++ b/src/datanode/Cargo.toml @@ -13,6 +13,7 @@ api = { path = "../api" } async-trait = "0.1" axum = "0.6.0-rc.2" axum-macros = "0.3.0-rc.1" +backon = "0.2" catalog = { path = "../catalog" } common-base = { path = "../common/base" } common-catalog = { path = "../common/catalog" } diff --git a/src/datanode/src/instance.rs b/src/datanode/src/instance.rs index 4646b525e5..dd94c9afc1 100644 --- a/src/datanode/src/instance.rs +++ b/src/datanode/src/instance.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use std::time::Duration; use std::{fs, path}; +use backon::ExponentialBackoff; use catalog::remote::MetaKvBackend; use catalog::CatalogManagerRef; use common_grpc::channel_manager::{ChannelConfig, ChannelManager}; @@ -26,7 +27,7 @@ use meta_client::client::{MetaClient, MetaClientBuilder}; use meta_client::MetaClientOpts; use mito::config::EngineConfig as TableEngineConfig; use mito::engine::MitoEngine; -use object_store::layers::LoggingLayer; +use object_store::layers::{LoggingLayer, MetricsLayer, RetryLayer, TracingLayer}; use object_store::services::fs::Builder; use object_store::{util, ObjectStore}; use query::query_engine::{QueryEngineFactory, QueryEngineRef}; @@ -189,7 +190,15 @@ pub(crate) async fn new_object_store(store_config: &ObjectStoreConfig) -> Result .build() .context(error::InitBackendSnafu { dir: &data_dir })?; - let object_store = ObjectStore::new(accessor).layer(LoggingLayer); // Add logging + let object_store = ObjectStore::new(accessor) + // Add retry + .layer(RetryLayer::new(ExponentialBackoff::default().with_jitter())) + // Add metrics + .layer(MetricsLayer) + // Add logging + .layer(LoggingLayer) + // Add tracing + .layer(TracingLayer); Ok(object_store) } diff --git a/src/object-store/Cargo.toml b/src/object-store/Cargo.toml index e7e63109e1..c85ae8c172 100644 --- a/src/object-store/Cargo.toml +++ b/src/object-store/Cargo.toml @@ -6,7 +6,7 @@ license = "Apache-2.0" [dependencies] futures = { version = "0.3" } -opendal = "0.20" +opendal = { version = "0.20", features = ["layers-tracing", "layers-metrics"]} tokio = { version = "1.0", features = ["full"] } [dev-dependencies]