diff --git a/Cargo.lock b/Cargo.lock index d023d340d4..dd13e5a833 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4313,6 +4313,7 @@ dependencies = [ "tokio-util", "toml_edit", "tracing", + "tracing-utils", "url", "utils", "uuid", @@ -7850,6 +7851,7 @@ dependencies = [ "tracing", "tracing-error", "tracing-subscriber", + "tracing-utils", "walkdir", ] diff --git a/compute_tools/src/logger.rs b/compute_tools/src/logger.rs index a65614e94e..c36f302f99 100644 --- a/compute_tools/src/logger.rs +++ b/compute_tools/src/logger.rs @@ -24,7 +24,8 @@ pub async fn init_tracing_and_logging(default_log_level: &str) -> anyhow::Result .with_writer(std::io::stderr); // Initialize OpenTelemetry - let otlp_layer = tracing_utils::init_tracing("compute_ctl").await; + let otlp_layer = + tracing_utils::init_tracing("compute_ctl", tracing_utils::ExportConfig::default()).await; // Put it all together tracing_subscriber::registry() diff --git a/libs/tracing-utils/src/lib.rs b/libs/tracing-utils/src/lib.rs index 72f94d61e4..74992a7d03 100644 --- a/libs/tracing-utils/src/lib.rs +++ b/libs/tracing-utils/src/lib.rs @@ -21,7 +21,7 @@ //! .with_writer(std::io::stderr); //! //! // Initialize OpenTelemetry. Exports tracing spans as OpenTelemetry traces -//! let otlp_layer = tracing_utils::init_tracing("my_application").await; +//! let otlp_layer = tracing_utils::init_tracing("my_application", tracing_utils::ExportConfig::default()).await; //! //! // Put it all together //! tracing_subscriber::registry() @@ -38,8 +38,12 @@ pub mod http; use opentelemetry::KeyValue; use opentelemetry::trace::TracerProvider; -use tracing::Subscriber; +use opentelemetry_otlp::WithExportConfig; +pub use opentelemetry_otlp::{ExportConfig, Protocol}; +use tracing::level_filters::LevelFilter; +use tracing::{Dispatch, Subscriber}; use tracing_subscriber::Layer; +use tracing_subscriber::layer::SubscriberExt; use tracing_subscriber::registry::LookupSpan; /// Set up OpenTelemetry exporter, using configuration from environment variables. @@ -69,19 +73,28 @@ use tracing_subscriber::registry::LookupSpan; /// /// This doesn't block, but is marked as 'async' to hint that this must be called in /// asynchronous execution context. -pub async fn init_tracing(service_name: &str) -> Option> +pub async fn init_tracing( + service_name: &str, + export_config: ExportConfig, +) -> Option> where S: Subscriber + for<'span> LookupSpan<'span>, { if std::env::var("OTEL_SDK_DISABLED") == Ok("true".to_string()) { return None; }; - Some(init_tracing_internal(service_name.to_string())) + Some(init_tracing_internal( + service_name.to_string(), + export_config, + )) } /// Like `init_tracing`, but creates a separate tokio Runtime for the tracing /// tasks. -pub fn init_tracing_without_runtime(service_name: &str) -> Option> +pub fn init_tracing_without_runtime( + service_name: &str, + export_config: ExportConfig, +) -> Option> where S: Subscriber + for<'span> LookupSpan<'span>, { @@ -112,16 +125,22 @@ where )); let _guard = runtime.enter(); - Some(init_tracing_internal(service_name.to_string())) + Some(init_tracing_internal( + service_name.to_string(), + export_config, + )) } -fn init_tracing_internal(service_name: String) -> impl Layer +fn init_tracing_internal(service_name: String, export_config: ExportConfig) -> impl Layer where S: Subscriber + for<'span> LookupSpan<'span>, { - // Sets up exporter from the OTEL_EXPORTER_* environment variables. + // Sets up exporter from the provided [`ExportConfig`] parameter. + // If the endpoint is not specified, it is loaded from the + // OTEL_EXPORTER_OTLP_ENDPOINT environment variable. let exporter = opentelemetry_otlp::SpanExporter::builder() .with_http() + .with_export_config(export_config) .build() .expect("could not initialize opentelemetry exporter"); @@ -151,3 +170,51 @@ where pub fn shutdown_tracing() { opentelemetry::global::shutdown_tracer_provider(); } + +pub enum OtelEnablement { + Disabled, + Enabled { + service_name: String, + export_config: ExportConfig, + runtime: &'static tokio::runtime::Runtime, + }, +} + +pub struct OtelGuard { + pub dispatch: Dispatch, +} + +impl Drop for OtelGuard { + fn drop(&mut self) { + shutdown_tracing(); + } +} + +/// Initializes OTEL infrastructure for performance tracing according to the provided configuration +/// +/// Performance tracing is handled by a different [`tracing::Subscriber`]. This functions returns +/// an [`OtelGuard`] containing a [`tracing::Dispatch`] associated with a newly created subscriber. +/// Applications should use this dispatch for their performance traces. +/// +/// The lifetime of the guard should match taht of the application. On drop, it tears down the +/// OTEL infra. +pub fn init_performance_tracing(otel_enablement: OtelEnablement) -> Option { + let otel_subscriber = match otel_enablement { + OtelEnablement::Disabled => None, + OtelEnablement::Enabled { + service_name, + export_config, + runtime, + } => { + let otel_layer = runtime + .block_on(init_tracing(&service_name, export_config)) + .with_filter(LevelFilter::INFO); + let otel_subscriber = tracing_subscriber::registry().with(otel_layer); + let otel_dispatch = Dispatch::new(otel_subscriber); + + Some(otel_dispatch) + } + }; + + otel_subscriber.map(|dispatch| OtelGuard { dispatch }) +} diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index ac44300a51..4180602ac7 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -42,6 +42,7 @@ toml_edit = { workspace = true, features = ["serde"] } tracing.workspace = true tracing-error.workspace = true tracing-subscriber = { workspace = true, features = ["json", "registry"] } +tracing-utils.workspace = true rand.workspace = true scopeguard.workspace = true strum.workspace = true diff --git a/libs/utils/src/logging.rs b/libs/utils/src/logging.rs index 881f1e765d..f37f05692a 100644 --- a/libs/utils/src/logging.rs +++ b/libs/utils/src/logging.rs @@ -165,6 +165,7 @@ pub fn init( }; log_layer.with_filter(rust_log_env_filter()) }); + let r = r.with( TracingEventCountLayer(&TRACING_EVENT_COUNT_METRIC).with_filter(rust_log_env_filter()), ); diff --git a/pageserver/Cargo.toml b/pageserver/Cargo.toml index a372be5044..d17a19ce65 100644 --- a/pageserver/Cargo.toml +++ b/pageserver/Cargo.toml @@ -70,6 +70,7 @@ tokio-stream.workspace = true tokio-util.workspace = true toml_edit = { workspace = true, features = [ "serde" ] } tracing.workspace = true +tracing-utils.workspace = true url.workspace = true walkdir.workspace = true metrics.workspace = true diff --git a/pageserver/compaction/tests/tests.rs b/pageserver/compaction/tests/tests.rs index bd8b54a286..565f66ce1a 100644 --- a/pageserver/compaction/tests/tests.rs +++ b/pageserver/compaction/tests/tests.rs @@ -12,7 +12,7 @@ pub(crate) fn setup_logging() { logging::TracingErrorLayerEnablement::EnableWithRustLogFilter, logging::Output::Stdout, ) - .expect("Failed to init test logging") + .expect("Failed to init test logging"); }); } diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index c4af0d5d41..4d30a6358b 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -111,6 +111,7 @@ fn main() -> anyhow::Result<()> { } else { TracingErrorLayerEnablement::Disabled }; + logging::init( conf.log_format, tracing_error_layer_enablement, diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 62e1cdac0c..2bce56345a 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -5754,7 +5754,7 @@ pub(crate) mod harness { logging::TracingErrorLayerEnablement::EnableWithRustLogFilter, logging::Output::Stdout, ) - .expect("Failed to init test logging") + .expect("Failed to init test logging"); }); } diff --git a/proxy/src/logging.rs b/proxy/src/logging.rs index 6f9845fd6e..454fe81357 100644 --- a/proxy/src/logging.rs +++ b/proxy/src/logging.rs @@ -46,7 +46,8 @@ pub async fn init() -> anyhow::Result { .expect("this should be a valid filter directive"), ); - let otlp_layer = tracing_utils::init_tracing("proxy").await; + let otlp_layer = + tracing_utils::init_tracing("proxy", tracing_utils::ExportConfig::default()).await; let json_log_layer = if logfmt == LogFormat::Json { Some(JsonLoggingLayer::new(