utils: optionally enable otel tracing in common logging utils

This patch augments `utils::logging::init` with the ability to set up
OTEL tracing infrastructure. The end goal is for the pageserver to use
this in order to export perf traces.

Note that an entirely different tracing subscriber is used. This is to
avoid interference with the existing tracing set-up.

For now, no service uses this functionality.

Cargo.lock

@@ -4301,6 +4301,7 @@ dependencies = [
  "tokio-util",
  "toml_edit",
  "tracing",
+ "tracing-utils",
  "url",
  "utils",
  "uuid",
@@ -7837,6 +7838,7 @@ dependencies = [
  "tracing",
  "tracing-error",
  "tracing-subscriber",
+ "tracing-utils",
  "walkdir",
 ]
 

compute_tools/src/bin/fast_import.rs

@@ -592,6 +592,7 @@ pub(crate) async fn main() -> anyhow::Result<()> {
     utils::logging::init(
         utils::logging::LogFormat::Json,
         utils::logging::TracingErrorLayerEnablement::EnableWithRustLogFilter,
+        utils::logging::OtelEnablement::Disabled,
         utils::logging::Output::Stdout,
     )?;
 

libs/desim/tests/reliable_copy_test.rs

@@ -158,6 +158,7 @@ mod reliable_copy_test {
         utils::logging::init(
             utils::logging::LogFormat::Test,
             utils::logging::TracingErrorLayerEnablement::Disabled,
+            utils::logging::OtelEnablement::Disabled,
             utils::logging::Output::Stdout,
         )
         .expect("logging init failed");

libs/remote_storage/tests/common/mod.rs

@@ -208,6 +208,7 @@ pub(crate) fn ensure_logging_ready() {
         utils::logging::init(
             utils::logging::LogFormat::Test,
             utils::logging::TracingErrorLayerEnablement::Disabled,
+            utils::logging::OtelEnablement::Disabled,
             utils::logging::Output::Stdout,
         )
         .expect("logging init failed");

libs/utils/Cargo.toml

@@ -42,6 +42,7 @@ toml_edit = { workspace = true, features = ["serde"] }
 tracing.workspace = true
 tracing-error.workspace = true
 tracing-subscriber = { workspace = true, features = ["json", "registry"] }
+tracing-utils.workspace = true
 rand.workspace = true
 scopeguard.workspace = true
 strum.workspace = true

libs/utils/src/logging.rs

@@ -7,7 +7,9 @@ use metrics::{IntCounter, IntCounterVec};
 use once_cell::sync::Lazy;
 use strum_macros::{EnumString, VariantNames};
 use tokio::time::Instant;
+use tracing::Dispatch;
 use tracing::info;
+use tracing::level_filters::LevelFilter;
 
 /// Logs a critical error, similarly to `tracing::error!`. This will:
 ///
@@ -125,6 +127,15 @@ pub enum TracingErrorLayerEnablement {
     EnableWithRustLogFilter,
 }
 
+pub enum OtelEnablement {
+    Disabled,
+    Enabled {
+        service_name: String,
+        export_config: tracing_utils::ExportConfig,
+        runtime: &'static tokio::runtime::Runtime,
+    },
+}
+
 /// Where the logging should output to.
 #[derive(Clone, Copy)]
 pub enum Output {
@@ -132,11 +143,22 @@ pub enum Output {
     Stderr,
 }
 
+pub struct OtelGuard {
+    pub dispatch: Dispatch,
+}
+
+impl Drop for OtelGuard {
+    fn drop(&mut self) {
+        tracing_utils::shutdown_tracing();
+    }
+}
+
 pub fn init(
     log_format: LogFormat,
     tracing_error_layer_enablement: TracingErrorLayerEnablement,
+    otel_enablement: OtelEnablement,
     output: Output,
-) -> anyhow::Result<()> {
+) -> anyhow::Result<Option<OtelGuard>> {
     // We fall back to printing all spans at info-level or above if
     // the RUST_LOG environment variable is not set.
     let rust_log_env_filter = || {
@@ -165,6 +187,7 @@ pub fn init(
         };
         log_layer.with_filter(rust_log_env_filter())
     });
+
     let r = r.with(
         TracingEventCountLayer(&TRACING_EVENT_COUNT_METRIC).with_filter(rust_log_env_filter()),
     );
@@ -175,7 +198,26 @@ pub fn init(
         TracingErrorLayerEnablement::Disabled => r.init(),
     }
 
-    Ok(())
+    let otel_subscriber = match otel_enablement {
+        OtelEnablement::Disabled => None,
+        OtelEnablement::Enabled {
+            service_name,
+            export_config,
+            runtime,
+        } => {
+            let otel_layer = runtime
+                .block_on(tracing_utils::init_tracing(&service_name, export_config))
+                .with_filter(LevelFilter::INFO);
+            let otel_subscriber = tracing_subscriber::registry().with(otel_layer);
+            let otel_dispatch = Dispatch::new(otel_subscriber);
+
+            Some(otel_dispatch)
+        }
+    };
+
+    let otel_guard = otel_subscriber.map(|dispatch| OtelGuard { dispatch });
+
+    Ok(otel_guard)
 }
 
 /// Disable the default rust panic hook by using `set_hook`.

pageserver/Cargo.toml

@@ -66,6 +66,7 @@ tokio-stream.workspace = true
 tokio-util.workspace = true
 toml_edit = { workspace = true, features = [ "serde" ] }
 tracing.workspace = true
+tracing-utils.workspace = true
 url.workspace = true
 walkdir.workspace = true
 metrics.workspace = true

pageserver/compaction/tests/tests.rs

@@ -10,9 +10,10 @@ pub(crate) fn setup_logging() {
         logging::init(
             logging::LogFormat::Test,
             logging::TracingErrorLayerEnablement::EnableWithRustLogFilter,
+            utils::logging::OtelEnablement::Disabled,
             logging::Output::Stdout,
         )
-        .expect("Failed to init test logging")
+        .expect("Failed to init test logging");
     });
 }
 

pageserver/ctl/src/main.rs

@@ -117,6 +117,7 @@ async fn main() -> anyhow::Result<()> {
     logging::init(
         LogFormat::Plain,
         TracingErrorLayerEnablement::EnableWithRustLogFilter,
+        utils::logging::OtelEnablement::Disabled,
         logging::Output::Stdout,
     )?;
 

pageserver/pagebench/src/main.rs

@@ -35,6 +35,7 @@ fn main() {
     logging::init(
         logging::LogFormat::Plain,
         logging::TracingErrorLayerEnablement::Disabled,
+        utils::logging::OtelEnablement::Disabled,
         logging::Output::Stderr,
     )
     .unwrap();

pageserver/src/bin/pageserver.rs

@@ -110,9 +110,11 @@ fn main() -> anyhow::Result<()> {
     } else {
         TracingErrorLayerEnablement::Disabled
     };
+
     logging::init(
         conf.log_format,
         tracing_error_layer_enablement,
+        utils::logging::OtelEnablement::Disabled,
         logging::Output::Stdout,
     )?;
 

pageserver/src/tenant.rs

@@ -5718,9 +5718,10 @@ pub(crate) mod harness {
                 // enable it in case the tests exercise code paths that use
                 // debug_assert_current_span_has_tenant_and_timeline_id
                 logging::TracingErrorLayerEnablement::EnableWithRustLogFilter,
+                utils::logging::OtelEnablement::Disabled,
                 logging::Output::Stdout,
             )
-            .expect("Failed to init test logging")
+            .expect("Failed to init test logging");
         });
     }
 

safekeeper/src/bin/safekeeper.rs

@@ -255,6 +255,7 @@ async fn main() -> anyhow::Result<()> {
     logging::init(
         LogFormat::from_config(&args.log_format)?,
         logging::TracingErrorLayerEnablement::Disabled,
+        utils::logging::OtelEnablement::Disabled,
         logging::Output::Stdout,
     )?;
     logging::replace_panic_hook_with_tracing_panic_hook().forget();

storage_broker/src/bin/storage_broker.rs

@@ -643,6 +643,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
     logging::init(
         LogFormat::from_config(&args.log_format)?,
         logging::TracingErrorLayerEnablement::Disabled,
+        utils::logging::OtelEnablement::Disabled,
         logging::Output::Stdout,
     )?;
     logging::replace_panic_hook_with_tracing_panic_hook().forget();

storage_controller/src/main.rs

@@ -240,6 +240,7 @@ fn main() -> anyhow::Result<()> {
     logging::init(
         LogFormat::Plain,
         logging::TracingErrorLayerEnablement::Disabled,
+        utils::logging::OtelEnablement::Disabled,
         logging::Output::Stdout,
     )?;