refactor: add tracing options in xOptions (#3919)

* refactor: add tracing options in {DatanodeOptions, FrontendOptions, MetasrvOptions, StandaloneOptions}

* ci: fix integration error

* refactor: minor modification for initialization of tracing options

config/config.md

@@ -127,6 +127,8 @@
 | `export_metrics.remote_write` | -- | -- | -- |
 | `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
+| `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
+| `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
 
 
 ## Cluster Mode
@@ -203,6 +205,8 @@
 | `export_metrics.remote_write` | -- | -- | -- |
 | `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
+| `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
+| `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
 
 
 ### Metasrv
@@ -259,6 +263,8 @@
 | `export_metrics.remote_write` | -- | -- | -- |
 | `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
+| `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
+| `tracing.tokio_console_addr` | String | `None` | The tokio console address. |
 
 
 ### Datanode
@@ -370,3 +376,5 @@
 | `export_metrics.remote_write` | -- | -- | -- |
 | `export_metrics.remote_write.url` | String | `""` | The url the metrics send to. The url example can be: `http://127.0.0.1:4000/v1/prometheus/write?db=information_schema`. |
 | `export_metrics.remote_write.headers` | InlineTable | -- | HTTP headers of Prometheus remote-write carry. |
+| `tracing` | -- | -- | The tracing options. Only effect when compiled with `tokio-console` feature. |
+| `tracing.tokio_console_addr` | String | `None` | The tokio console address. |

config/datanode.example.toml

@@ -428,3 +428,9 @@ url = ""
 
 ## HTTP headers of Prometheus remote-write carry.
 headers = { }
+
+## The tracing options. Only effect when compiled with `tokio-console` feature.
+[tracing]
+## The tokio console address.
+## +toml2docs:none-default
+tokio_console_addr = "127.0.0.1"

config/frontend.example.toml

@@ -186,3 +186,9 @@ url = ""
 
 ## HTTP headers of Prometheus remote-write carry.
 headers = { }
+
+## The tracing options. Only effect when compiled with `tokio-console` feature.
+[tracing]
+## The tokio console address.
+## +toml2docs:none-default
+tokio_console_addr = "127.0.0.1"

config/metasrv.example.toml

@@ -141,3 +141,9 @@ url = ""
 
 ## HTTP headers of Prometheus remote-write carry.
 headers = { }
+
+## The tracing options. Only effect when compiled with `tokio-console` feature.
+[tracing]
+## The tokio console address.
+## +toml2docs:none-default
+tokio_console_addr = "127.0.0.1"

config/standalone.example.toml

@@ -471,3 +471,9 @@ url = ""
 
 ## HTTP headers of Prometheus remote-write carry.
 headers = { }
+
+## The tracing options. Only effect when compiled with `tokio-console` feature.
+[tracing]
+## The tokio console address.
+## +toml2docs:none-default
+tokio_console_addr = "127.0.0.1"

src/cmd/src/bin/greptime.rs

@@ -128,7 +128,7 @@ async fn start(cli: Command) -> Result<()> {
     let _guard = common_telemetry::init_global_logging(
         &app_name,
         opts.logging_options(),
-        cli.global_options.tracing_options(),
+        &cli.global_options.tracing_options(),
         opts.node_id(),
     );
 

src/cmd/src/datanode.rs

@@ -19,6 +19,7 @@ use async_trait::async_trait;
 use catalog::kvbackend::MetaKvBackend;
 use clap::Parser;
 use common_telemetry::info;
+use common_telemetry::logging::TracingOptions;
 use common_wal::config::DatanodeWalConfig;
 use datanode::config::DatanodeOptions;
 use datanode::datanode::{Datanode, DatanodeBuilder};
@@ -146,6 +147,11 @@ impl StartCommand {
             opts.logging.level.clone_from(&global_options.log_level);
         }
 
+        opts.tracing = TracingOptions {
+            #[cfg(feature = "tokio-console")]
+            tokio_console_addr: global_options.tokio_console_addr.clone(),
+        };
+
         if let Some(addr) = &self.rpc_addr {
             opts.rpc_addr.clone_from(addr);
         }

src/cmd/src/frontend.rs

@@ -23,6 +23,7 @@ use client::client_manager::DatanodeClients;
 use common_meta::heartbeat::handler::parse_mailbox_message::ParseMailboxMessageHandler;
 use common_meta::heartbeat::handler::HandlerGroupExecutor;
 use common_telemetry::info;
+use common_telemetry::logging::TracingOptions;
 use common_time::timezone::set_default_timezone;
 use frontend::frontend::FrontendOptions;
 use frontend::heartbeat::handler::invalidate_table_cache::InvalidateTableCacheHandler;
@@ -162,6 +163,11 @@ impl StartCommand {
             opts.logging.level.clone_from(&global_options.log_level);
         }
 
+        opts.tracing = TracingOptions {
+            #[cfg(feature = "tokio-console")]
+            tokio_console_addr: global_options.tokio_console_addr.clone(),
+        };
+
         let tls_opts = TlsOption::new(
             self.tls_mode.clone(),
             self.tls_cert_path.clone(),

src/cmd/src/metasrv.rs

@@ -17,6 +17,7 @@ use std::time::Duration;
 use async_trait::async_trait;
 use clap::Parser;
 use common_telemetry::info;
+use common_telemetry::logging::TracingOptions;
 use meta_srv::bootstrap::MetasrvInstance;
 use meta_srv::metasrv::MetasrvOptions;
 use snafu::ResultExt;
@@ -141,6 +142,11 @@ impl StartCommand {
             opts.logging.level.clone_from(&global_options.log_level);
         }
 
+        opts.tracing = TracingOptions {
+            #[cfg(feature = "tokio-console")]
+            tokio_console_addr: global_options.tokio_console_addr.clone(),
+        };
+
         if let Some(addr) = &self.bind_addr {
             opts.bind_addr.clone_from(addr);
         }

src/cmd/src/standalone.rs

@@ -35,7 +35,7 @@ use common_meta::sequence::SequenceBuilder;
 use common_meta::wal_options_allocator::{WalOptionsAllocator, WalOptionsAllocatorRef};
 use common_procedure::ProcedureManagerRef;
 use common_telemetry::info;
-use common_telemetry::logging::LoggingOptions;
+use common_telemetry::logging::{LoggingOptions, TracingOptions};
 use common_time::timezone::set_default_timezone;
 use common_wal::config::StandaloneWalConfig;
 use datanode::config::{DatanodeOptions, ProcedureConfig, RegionEngineConfig, StorageConfig};
@@ -124,6 +124,7 @@ pub struct StandaloneOptions {
     /// Options for different store engines.
     pub region_engine: Vec<RegionEngineConfig>,
     pub export_metrics: ExportMetricsOption,
+    pub tracing: TracingOptions,
 }
 
 impl StandaloneOptions {
@@ -156,6 +157,7 @@ impl Default for StandaloneOptions {
                 RegionEngineConfig::Mito(MitoConfig::default()),
                 RegionEngineConfig::File(FileEngineConfig::default()),
             ],
+            tracing: TracingOptions::default(),
         }
     }
 }
@@ -302,6 +304,11 @@ impl StartCommand {
             opts.logging.level.clone_from(&global_options.log_level);
         }
 
+        opts.tracing = TracingOptions {
+            #[cfg(feature = "tokio-console")]
+            tokio_console_addr: global_options.tokio_console_addr.clone(),
+        };
+
         let tls_opts = TlsOption::new(
             self.tls_mode.clone(),
             self.tls_cert_path.clone(),

src/common/telemetry/src/logging.rs

@@ -72,7 +72,7 @@ impl Default for LoggingOptions {
     }
 }
 
-#[derive(Default)]
+#[derive(Default, Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
 pub struct TracingOptions {
     #[cfg(feature = "tokio-console")]
     pub tokio_console_addr: Option<String>,
@@ -104,7 +104,7 @@ pub fn init_default_ut_logging() {
         *g = Some(init_global_logging(
             "unittest",
             &opts,
-            TracingOptions::default(),
+            &TracingOptions::default(),
             None
         ));
 
@@ -121,7 +121,7 @@ const DEFAULT_LOG_TARGETS: &str = "info";
 pub fn init_global_logging(
     app_name: &str,
     opts: &LoggingOptions,
-    tracing_opts: TracingOptions,
+    tracing_opts: &TracingOptions,
     node_id: Option<String>,
 ) -> Vec<WorkerGuard> {
     let mut guards = vec![];

src/datanode/src/config.rs

@@ -20,7 +20,7 @@ use common_grpc::channel_manager::{
     DEFAULT_MAX_GRPC_RECV_MESSAGE_SIZE, DEFAULT_MAX_GRPC_SEND_MESSAGE_SIZE,
 };
 pub use common_procedure::options::ProcedureConfig;
-use common_telemetry::logging::LoggingOptions;
+use common_telemetry::logging::{LoggingOptions, TracingOptions};
 use common_wal::config::DatanodeWalConfig;
 use file_engine::config::EngineConfig as FileEngineConfig;
 use meta_client::MetaClientOptions;
@@ -234,6 +234,7 @@ pub struct DatanodeOptions {
     pub logging: LoggingOptions,
     pub enable_telemetry: bool,
     pub export_metrics: ExportMetricsOption,
+    pub tracing: TracingOptions,
 }
 
 impl Default for DatanodeOptions {
@@ -260,6 +261,7 @@ impl Default for DatanodeOptions {
             heartbeat: HeartbeatOptions::datanode_default(),
             enable_telemetry: true,
             export_metrics: ExportMetricsOption::default(),
+            tracing: TracingOptions::default(),
         }
     }
 }

src/frontend/src/frontend.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-use common_telemetry::logging::LoggingOptions;
+use common_telemetry::logging::{LoggingOptions, TracingOptions};
 use meta_client::MetaClientOptions;
 use serde::{Deserialize, Serialize};
 use servers::export_metrics::ExportMetricsOption;
@@ -47,6 +47,7 @@ pub struct FrontendOptions {
     pub datanode: DatanodeOptions,
     pub user_provider: Option<String>,
     pub export_metrics: ExportMetricsOption,
+    pub tracing: TracingOptions,
 }
 
 impl Default for FrontendOptions {
@@ -69,6 +70,7 @@ impl Default for FrontendOptions {
             datanode: DatanodeOptions::default(),
             user_provider: None,
             export_metrics: ExportMetricsOption::default(),
+            tracing: TracingOptions::default(),
         }
     }
 }

src/meta-srv/src/metasrv.rs

@@ -31,7 +31,7 @@ use common_meta::wal_options_allocator::WalOptionsAllocatorRef;
 use common_meta::{distributed_time_constants, ClusterId};
 use common_procedure::options::ProcedureConfig;
 use common_procedure::ProcedureManagerRef;
-use common_telemetry::logging::LoggingOptions;
+use common_telemetry::logging::{LoggingOptions, TracingOptions};
 use common_telemetry::{error, info, warn};
 use common_wal::config::MetasrvWalConfig;
 use serde::{Deserialize, Serialize};
@@ -109,6 +109,8 @@ pub struct MetasrvOptions {
     /// limit the number of operations in a txn because an infinitely large txn could
     /// potentially block other operations.
     pub max_txn_ops: usize,
+    /// The tracing options.
+    pub tracing: TracingOptions,
 }
 
 impl MetasrvOptions {
@@ -146,6 +148,7 @@ impl Default for MetasrvOptions {
             export_metrics: ExportMetricsOption::default(),
             store_key_prefix: String::new(),
             max_txn_ops: 128,
+            tracing: TracingOptions::default(),
         }
     }
 }

tests-integration/tests/http.rs

@@ -835,7 +835,9 @@ type = "time_series"
 
 [export_metrics]
 enable = false
-write_interval = "30s""#,
+write_interval = "30s"
+
+[tracing]"#,
         store_type
     )
     .trim()