From 95809cd3da0f1760fcbe2b22482be599e2e49a7d Mon Sep 17 00:00:00 2001 From: Weny Xu Date: Tue, 17 Nov 2026 18:19:29 +0000 Subject: [PATCH] fix: return file logging initialization errors instead of panicking (#10478) Signed-off-by: WenyXu --- Cargo.lock | 2 + config/config.md | 10 +- config/datanode.example.toml | 2 + config/flownode.example.toml | 2 + config/frontend.example.toml | 2 + config/metasrv.example.toml | 2 + config/standalone.example.toml | 2 + src/cmd/src/cli.rs | 3 +- src/cmd/src/datanode/builder.rs | 3 +- src/cmd/src/error.rs | 8 + src/cmd/src/flownode.rs | 3 +- src/cmd/src/frontend.rs | 3 +- src/cmd/src/metasrv.rs | 3 +- src/cmd/src/standalone.rs | 3 +- src/common/telemetry/Cargo.toml | 2 + src/common/telemetry/src/logging.rs | 193 +++++++++++++++--- .../telemetry/src/logging/file_retention.rs | 41 ++-- 17 files changed, 229 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d8822ef5c38..c716ca8c81a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2931,6 +2931,7 @@ dependencies = [ "backtrace", "common-base", "common-error", + "common-macro", "common-version", "console-subscriber", "greptime-proto", @@ -2945,6 +2946,7 @@ dependencies = [ "prometheus 0.14.0", "serde", "serde_json", + "snafu 0.8.6", "tempfile", "tokio", "tracing", diff --git a/config/config.md b/config/config.md index 27aeabab52c..16e98f77aea 100644 --- a/config/config.md +++ b/config/config.md @@ -260,7 +260,7 @@ | `logging.enable_file_logging` | Bool | `true` | Whether to write logs to files in `dir`. | | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. | | `logging.max_log_files` | Integer | `720` | The maximum amount of log files. | -| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. | +| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
Requires symbolic links in the log directory; initialization fails if they cannot be created.
On Windows, enable Developer Mode or leave this at `0B`. | | `logging.enable_per_region_metrics` | Bool | `false` | Whether to enable per-region metrics.
Default to false. | | `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. | | `logging.otlp_headers` | -- | -- | Additional OTLP headers, only valid when using OTLP http | @@ -422,7 +422,7 @@ | `logging.enable_file_logging` | Bool | `true` | Whether to write logs to files in `dir`. | | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. | | `logging.max_log_files` | Integer | `720` | The maximum amount of log files. | -| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. | +| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
Requires symbolic links in the log directory; initialization fails if they cannot be created.
On Windows, enable Developer Mode or leave this at `0B`. | | `logging.enable_per_region_metrics` | Bool | `false` | Whether to enable per-region metrics.
Default to false. | | `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. | | `logging.otlp_headers` | -- | -- | Additional OTLP headers, only valid when using OTLP http | @@ -541,7 +541,7 @@ | `logging.enable_file_logging` | Bool | `true` | Whether to write logs to files in `dir`. | | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. | | `logging.max_log_files` | Integer | `720` | The maximum amount of log files. | -| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. | +| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
Requires symbolic links in the log directory; initialization fails if they cannot be created.
On Windows, enable Developer Mode or leave this at `0B`. | | `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. | | `logging.otlp_headers` | -- | -- | Additional OTLP headers, only valid when using OTLP http | | `logging.tracing_sample_ratio` | -- | Unset | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 | @@ -748,7 +748,7 @@ | `logging.enable_file_logging` | Bool | `true` | Whether to write logs to files in `dir`. | | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. | | `logging.max_log_files` | Integer | `720` | The maximum amount of log files. | -| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. | +| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
Requires symbolic links in the log directory; initialization fails if they cannot be created.
On Windows, enable Developer Mode or leave this at `0B`. | | `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. | | `logging.otlp_headers` | -- | -- | Additional OTLP headers, only valid when using OTLP http | | `logging.tracing_sample_ratio` | -- | Unset | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 | @@ -809,7 +809,7 @@ | `logging.enable_file_logging` | Bool | `true` | Whether to write logs to files in `dir`. | | `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. | | `logging.max_log_files` | Integer | `720` | The maximum amount of log files. | -| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. | +| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
Requires symbolic links in the log directory; initialization fails if they cannot be created.
On Windows, enable Developer Mode or leave this at `0B`. | | `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. | | `logging.otlp_headers` | -- | -- | Additional OTLP headers, only valid when using OTLP http | | `logging.tracing_sample_ratio` | -- | Unset | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 | diff --git a/config/datanode.example.toml b/config/datanode.example.toml index 048a3b7a39d..342ceb0a17a 100644 --- a/config/datanode.example.toml +++ b/config/datanode.example.toml @@ -906,6 +906,8 @@ max_log_files = 720 ## The maximum total size of managed log files in `dir`. ## Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. +## Requires symbolic links in the log directory; initialization fails if they cannot be created. +## On Windows, enable Developer Mode or leave this at `0B`. max_log_dir_size = "0B" ## The OTLP tracing export protocol. Can be `grpc`/`http`. diff --git a/config/flownode.example.toml b/config/flownode.example.toml index fab31fed510..8faeb2cc227 100644 --- a/config/flownode.example.toml +++ b/config/flownode.example.toml @@ -126,6 +126,8 @@ max_log_files = 720 ## The maximum total size of managed log files in `dir`. ## Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. +## Requires symbolic links in the log directory; initialization fails if they cannot be created. +## On Windows, enable Developer Mode or leave this at `0B`. max_log_dir_size = "0B" ## The OTLP tracing export protocol. Can be `grpc`/`http`. diff --git a/config/frontend.example.toml b/config/frontend.example.toml index b8a73240345..d5638b66f58 100644 --- a/config/frontend.example.toml +++ b/config/frontend.example.toml @@ -464,6 +464,8 @@ max_log_files = 720 ## The maximum total size of managed log files in `dir`. ## Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. +## Requires symbolic links in the log directory; initialization fails if they cannot be created. +## On Windows, enable Developer Mode or leave this at `0B`. max_log_dir_size = "0B" ## Whether to enable per-region metrics. diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml index fe4c38d02f9..35ba3ae0a98 100644 --- a/config/metasrv.example.toml +++ b/config/metasrv.example.toml @@ -392,6 +392,8 @@ max_log_files = 720 ## The maximum total size of managed log files in `dir`. ## Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. +## Requires symbolic links in the log directory; initialization fails if they cannot be created. +## On Windows, enable Developer Mode or leave this at `0B`. max_log_dir_size = "0B" ## The OTLP tracing export protocol. Can be `grpc`/`http`. diff --git a/config/standalone.example.toml b/config/standalone.example.toml index 616b6f4f795..237f106992b 100644 --- a/config/standalone.example.toml +++ b/config/standalone.example.toml @@ -1110,6 +1110,8 @@ max_log_files = 720 ## The maximum total size of managed log files in `dir`. ## Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. +## Requires symbolic links in the log directory; initialization fails if they cannot be created. +## On Windows, enable Developer Mode or leave this at `0B`. max_log_dir_size = "0B" ## Whether to enable per-region metrics. diff --git a/src/cmd/src/cli.rs b/src/cmd/src/cli.rs index 3739d4cc4fc..92aca3a5625 100644 --- a/src/cmd/src/cli.rs +++ b/src/cmd/src/cli.rs @@ -77,7 +77,8 @@ impl Command { &TracingOptions::default(), None, None, - ); + ) + .context(crate::error::InitLoggingSnafu)?; let tool = self.cmd.build().await.context(error::BuildCliSnafu)?; let instance = Instance { diff --git a/src/cmd/src/datanode/builder.rs b/src/cmd/src/datanode/builder.rs index 6df7ac728e8..fc933a9c5be 100644 --- a/src/cmd/src/datanode/builder.rs +++ b/src/cmd/src/datanode/builder.rs @@ -67,7 +67,8 @@ impl InstanceBuilder { &dn_opts.tracing, dn_opts.node_id.map(|x| x.to_string()), None, - ); + ) + .context(crate::error::InitLoggingSnafu)?; common_runtime::init_datanode_runtimes(&opts.runtime); diff --git a/src/cmd/src/error.rs b/src/cmd/src/error.rs index 7b0decf05ca..84553439ab1 100644 --- a/src/cmd/src/error.rs +++ b/src/cmd/src/error.rs @@ -23,6 +23,13 @@ use snafu::{Location, Snafu}; #[snafu(visibility(pub))] #[stack_trace_debug] pub enum Error { + #[snafu(display("Failed to initialize logging"))] + InitLogging { + #[snafu(implicit)] + location: Location, + source: common_telemetry::logging::InitLoggingError, + }, + #[snafu(display("Failed to install ring crypto provider: {}", msg))] InitTlsProvider { #[snafu(implicit)] @@ -337,6 +344,7 @@ pub type Result = std::result::Result; impl ErrorExt for Error { fn status_code(&self) -> StatusCode { match self { + Error::InitLogging { source, .. } => source.status_code(), Error::StartDatanode { source, .. } => source.status_code(), Error::StartFrontend { source, .. } => source.status_code(), Error::ShutdownDatanode { source, .. } => source.status_code(), diff --git a/src/cmd/src/flownode.rs b/src/cmd/src/flownode.rs index 0dfcf85e2e7..91faf313dc3 100644 --- a/src/cmd/src/flownode.rs +++ b/src/cmd/src/flownode.rs @@ -251,7 +251,8 @@ impl StartCommand { &opts.component.tracing, opts.component.node_id.map(|x| x.to_string()), None, - ); + ) + .context(crate::error::InitLoggingSnafu)?; common_runtime::init_global_runtimes(&opts.runtime); diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs index 4348d55799d..7a430d7619d 100644 --- a/src/cmd/src/frontend.rs +++ b/src/cmd/src/frontend.rs @@ -384,7 +384,8 @@ impl StartCommand { &opts.component.tracing, opts.component.node_id.clone(), Some(&opts.component.slow_query), - ); + ) + .context(crate::error::InitLoggingSnafu)?; common_runtime::init_global_runtimes(&opts.runtime); diff --git a/src/cmd/src/metasrv.rs b/src/cmd/src/metasrv.rs index 369f5c428d1..7df4b92ab00 100644 --- a/src/cmd/src/metasrv.rs +++ b/src/cmd/src/metasrv.rs @@ -324,7 +324,8 @@ impl StartCommand { &opts.component.tracing, None, None, - ); + ) + .context(crate::error::InitLoggingSnafu)?; common_runtime::init_global_runtimes(&opts.runtime); diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs index 9dbcb89abc4..934429517f4 100644 --- a/src/cmd/src/standalone.rs +++ b/src/cmd/src/standalone.rs @@ -498,7 +498,8 @@ impl StartCommand { &opts.component.tracing, None, Some(&opts.component.slow_query), - ); + ) + .context(crate::error::InitLoggingSnafu)?; common_runtime::init_standalone_runtimes(&opts.runtime); diff --git a/src/common/telemetry/Cargo.toml b/src/common/telemetry/Cargo.toml index 1e146a5a75e..33be63ed92e 100644 --- a/src/common/telemetry/Cargo.toml +++ b/src/common/telemetry/Cargo.toml @@ -15,6 +15,7 @@ workspace = true backtrace = "0.3" common-base.workspace = true common-error.workspace = true +common-macro.workspace = true common-version.workspace = true console-subscriber = { version = "0.1", optional = true } greptime-proto.workspace = true @@ -31,6 +32,7 @@ parking_lot.workspace = true prometheus.workspace = true serde.workspace = true serde_json.workspace = true +snafu.workspace = true tokio.workspace = true tracing = "0.1" tracing-appender.workspace = true diff --git a/src/common/telemetry/src/logging.rs b/src/common/telemetry/src/logging.rs index 491b84f3fab..5154f7dc67e 100644 --- a/src/common/telemetry/src/logging.rs +++ b/src/common/telemetry/src/logging.rs @@ -15,6 +15,7 @@ //! logging stuffs, inspired by databend mod file_retention; +use std::any::Any; use std::collections::HashMap; use std::env; use std::io::IsTerminal; @@ -24,6 +25,9 @@ use std::time::Duration; use common_base::readable_size::ReadableSize; use common_base::serde::empty_string_as_default; +use common_error::ext::ErrorExt; +use common_error::status_code::StatusCode; +use common_macro::stack_trace_debug; use file_retention::{DirectoryRetention, LogFileKind, build_file_appender}; use once_cell::sync::{Lazy, OnceCell}; use opentelemetry::trace::TracerProvider; @@ -33,6 +37,7 @@ use opentelemetry_sdk::propagation::TraceContextPropagator; use opentelemetry_sdk::trace::{Sampler, Tracer}; use opentelemetry_semantic_conventions::resource; use serde::{Deserialize, Serialize}; +use snafu::{Location, Snafu}; use tracing::callsite; use tracing::metadata::LevelFilter; use tracing_appender::non_blocking::{NonBlocking, WorkerGuard}; @@ -45,6 +50,31 @@ use tracing_subscriber::{EnvFilter, Registry, filter}; use crate::tracing_sampler::{TracingSampleOptions, create_sampler}; +/// Failure to initialize a file appender. +#[derive(Snafu)] +#[stack_trace_debug] +pub enum InitLoggingError { + #[snafu(display("Failed to initialize file logging in {directory}: {error}.{hint}"))] + FileAppender { + directory: String, + #[snafu(source)] + error: tracing_appender::rolling::InitError, + hint: &'static str, + #[snafu(implicit)] + location: Location, + }, +} + +impl ErrorExt for InitLoggingError { + fn status_code(&self) -> StatusCode { + StatusCode::InvalidArguments + } + + fn as_any(&self) -> &dyn Any { + self + } +} + /// The default endpoint when use gRPC exporter protocol. pub const DEFAULT_OTLP_GRPC_ENDPOINT: &str = "http://localhost:4317"; @@ -474,13 +504,16 @@ pub fn init_default_ut_logging() { level: Some(level), ..Default::default() }; - *g = Some(init_global_logging( - "unittest", - &opts, - &TracingOptions::default(), - None, - None, - )); + *g = Some( + init_global_logging( + "unittest", + &opts, + &TracingOptions::default(), + None, + None, + ) + .expect("failed to initialize test logging"), + ); crate::info!("logs dir = {}", dir); }); @@ -498,15 +531,12 @@ pub fn init_global_logging( tracing_opts: &TracingOptions, node_id: Option, slow_query_opts: Option<&SlowQueryOptions>, -) -> Vec { - static START: Once = Once::new(); +) -> Result, InitLoggingError> { + static START: OnceCell<()> = OnceCell::new(); let mut guards = vec![]; let node_id = node_id.unwrap_or_else(|| "none".to_string()); - START.call_once(|| { - // Enable log compatible layer to convert log record to tracing span. - LogTracer::init().expect("log tracer must be valid"); - + START.get_or_try_init(|| -> Result<(), InitLoggingError> { let stdout_writer = if opts.append_stdout { let (writer, guard) = tracing_appender::non_blocking(std::io::stdout()); guards.push(guard); @@ -531,7 +561,7 @@ pub fn init_global_logging( // Configure the file logging layer with rolling policy. let file_logging_layer = if file_logging_enabled { let rolling_appender = - build_file_appender(opts, LogFileKind::Default, retention.as_ref()); + build_file_appender(opts, LogFileKind::Default, retention.as_ref())?; let (writer, guard) = tracing_appender::non_blocking(rolling_appender); guards.push(guard); @@ -543,7 +573,7 @@ pub fn init_global_logging( // Configure the error file logging layer with rolling policy. let err_file_logging_layer = if file_logging_enabled { let rolling_appender = - build_file_appender(opts, LogFileKind::Error, retention.as_ref()); + build_file_appender(opts, LogFileKind::Error, retention.as_ref())?; let (writer, guard) = tracing_appender::non_blocking(rolling_appender); guards.push(guard); @@ -555,14 +585,23 @@ pub fn init_global_logging( None }; - if slow_query_logging_enabled { + let slow_query_logger = if slow_query_logging_enabled { + Some(build_slow_query_logger( + opts, + stdout_writer, + retention.as_ref(), + &mut guards, + )?) + } else { + None + }; + + // Build all file appenders before installing any global logging state. + // A filesystem failure can then be returned without poisoning initialization. + LogTracer::init().expect("log tracer must be valid"); + if let Some(logger) = slow_query_logger { SLOW_QUERY_LOGGER - .set(build_slow_query_logger( - opts, - stdout_writer, - retention.as_ref(), - &mut guards, - )) + .set(logger) .expect("slow query logger already initialized"); } @@ -659,9 +698,10 @@ pub fn init_global_logging( tracing::subscriber::set_global_default(subscriber) .expect("error setting global tracing subscriber"); - }); + Ok(()) + })?; - guards + Ok(guards) } fn create_tracer(app_name: &str, node_id: &str, opts: &LoggingOptions) -> Tracer { @@ -747,12 +787,12 @@ fn build_slow_query_logger( stdout_writer: Option, retention: Option<&DirectoryRetention>, guards: &mut Vec, -) -> tracing::Dispatch { +) -> Result { let stdout_layer = stdout_writer.map(|writer| { build_logging_layer(writer, opts.log_format, std::io::stdout().is_terminal()) }); let file_layer = if opts.enable_file_logging && !opts.dir.is_empty() { - let rolling_appender = build_file_appender(opts, LogFileKind::SlowQuery, retention); + let rolling_appender = build_file_appender(opts, LogFileKind::SlowQuery, retention)?; let (writer, guard) = tracing_appender::non_blocking(rolling_appender); guards.push(guard); Some(build_logging_layer(writer, opts.log_format, false)) @@ -764,12 +804,12 @@ fn build_slow_query_logger( *metadata.level() == tracing::Level::INFO && metadata.fields().field("slow").is_some() }) .with_max_level_hint(LevelFilter::INFO); - tracing::Dispatch::new( + Ok(tracing::Dispatch::new( Registry::default() .with(filter) .with(stdout_layer) .with(file_layer), - ) + )) } fn build_logging_layer( @@ -802,6 +842,100 @@ mod tests { use super::*; + #[test] + fn test_file_logging_initialization_failure() { + const CHILD_ENV: &str = "GREPTIME_TEST_LOGGING_INIT_FAILURE"; + if env::var_os(CHILD_ENV).is_none() { + let output = std::process::Command::new(env::current_exe().unwrap()) + .args([ + "--exact", + "logging::tests::test_file_logging_initialization_failure", + "--nocapture", + ]) + .env(CHILD_ENV, "1") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + return; + } + + let dir = tempfile::tempdir().unwrap(); + let mut opts = LoggingOptions { + dir: dir.path().to_str().unwrap().to_string(), + max_log_dir_size: ReadableSize::mb(1), + append_stdout: false, + ..Default::default() + }; + let slow_opts = SlowQueryOptions { + enable: true, + record_type: SlowQueriesRecordType::Log, + ..Default::default() + }; + for link in [ + ".greptimedb.current", + ".greptimedb-err.current", + ".greptimedb-slow-queries.current", + ] { + let path = dir.path().join(link); + // A directory collision fails even on runners allowed to create symlinks. + std::fs::create_dir(&path).unwrap(); + let error = init_global_logging( + "test", + &opts, + &TracingOptions::default(), + None, + Some(&slow_opts), + ) + .unwrap_err(); + let message = error.to_string(); + assert!(message.contains("logging.max_log_dir_size"), "{message}"); + assert!(message.contains(&opts.dir), "{message}"); + assert!(message.contains("Developer Mode"), "{message}"); + assert!(LOG_RELOAD_HANDLE.get().is_none()); + assert!(SLOW_QUERY_LOGGER.get().is_none()); + std::fs::remove_dir(path).unwrap(); + for entry in std::fs::read_dir(dir.path()).unwrap() { + let entry = entry.unwrap(); + if entry.file_type().unwrap().is_symlink() { + std::fs::remove_file(entry.path()).unwrap(); + } + } + } + + // Disabling retention succeeds with the same collision and after failed attempts. + std::fs::create_dir(dir.path().join(".greptimedb.current")).unwrap(); + opts.max_log_dir_size = ReadableSize::default(); + let guards = init_global_logging( + "test", + &opts, + &TracingOptions::default(), + None, + Some(&slow_opts), + ) + .unwrap(); + crate::error!("logging works after initialization failures"); + drop(guards); + let error_log = std::fs::read_dir(dir.path()) + .unwrap() + .map(|entry| entry.unwrap().path()) + .find(|path| { + path.file_name() + .unwrap() + .to_string_lossy() + .starts_with("greptimedb-err.") + }) + .unwrap(); + assert!( + std::fs::read_to_string(error_log) + .unwrap() + .contains("logging works after initialization failures") + ); + } + #[test] fn test_slow_query_logging() { const CASE_ENV: &str = "GREPTIME_TEST_SLOW_QUERY_LOGGING"; @@ -915,7 +1049,8 @@ mod tests { &TracingOptions::default(), None, (case != "no_options").then_some(&slow_opts), - ); + ) + .unwrap(); for level in ["warn", "error", "off", "info"] { LOG_RELOAD_HANDLE diff --git a/src/common/telemetry/src/logging/file_retention.rs b/src/common/telemetry/src/logging/file_retention.rs index 0a79468851f..d2a962ad748 100644 --- a/src/common/telemetry/src/logging/file_retention.rs +++ b/src/common/telemetry/src/logging/file_retention.rs @@ -24,9 +24,10 @@ use std::time::SystemTime; use common_base::readable_size::ReadableSize; use parking_lot::Mutex; +use snafu::ResultExt; use tracing_appender::rolling::{RollingFileAppender, Rotation}; -use crate::logging::LoggingOptions; +use crate::logging::{FileAppenderSnafu, InitLoggingError, LoggingOptions}; /// A managed file-log kind. #[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] @@ -421,7 +422,7 @@ pub(crate) fn build_file_appender( opts: &LoggingOptions, kind: LogFileKind, retention: Option<&DirectoryRetention>, -) -> Box { +) -> Result, InitLoggingError> { // Directory retention owns size and count pruning so its index remains authoritative. let upstream_max_log_files = if retention.is_some() { 0 @@ -440,18 +441,24 @@ pub(crate) fn build_file_appender( // before creating it, so a missing directory prints a misleading error. If this fails, // `build` reports it. let _ = fs::create_dir_all(&opts.dir); - let appender = builder.build(&opts.dir).unwrap_or_else(|error| { - panic!( - "initializing rolling file appender at {} failed: {}", - opts.dir, error - ) - }); + let appender = builder.build(&opts.dir).with_context(|_| FileAppenderSnafu { + directory: opts.dir.clone(), + hint: if retention.is_some() { + " logging.max_log_dir_size requires creating symbolic links in this directory. Check directory permissions and conflicting paths; on Windows, enable Developer Mode or set logging.max_log_dir_size = \"0B\" to disable size-based retention." + } else { + "" + }, + })?; if let Some(retention) = retention { retention.register(kind); - Box::new(RetentionAppender::new(appender, retention.clone(), kind)) + Ok(Box::new(RetentionAppender::new( + appender, + retention.clone(), + kind, + ))) } else { - Box::new(appender) + Ok(Box::new(appender)) } } @@ -521,7 +528,7 @@ mod tests { }; let retention = DirectoryRetention::new(directory.path(), ReadableSize::mb(1), 0).unwrap(); - let _appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention)); + let _appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention)).unwrap(); let symlink = directory.path().join(".greptimedb.current"); assert!(symlink.is_symlink()); @@ -542,7 +549,8 @@ mod tests { ..Default::default() }; let retention = DirectoryRetention::new(directory.path(), ReadableSize(1024), 0).unwrap(); - let mut appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention)); + let mut appender = + build_file_appender(&opts, LogFileKind::Default, Some(&retention)).unwrap(); retention.initialize(); appender.write_all(b"current").unwrap(); @@ -570,7 +578,8 @@ mod tests { ..Default::default() }; let retention = DirectoryRetention::new(directory.path(), ReadableSize(1024), 0).unwrap(); - let mut appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention)); + let mut appender = + build_file_appender(&opts, LogFileKind::Default, Some(&retention)).unwrap(); retention.initialize(); appender.write_all(b"current").unwrap(); @@ -593,7 +602,8 @@ mod tests { ..Default::default() }; let retention = DirectoryRetention::new(directory.path(), ReadableSize::gb(1), 2).unwrap(); - let mut appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention)); + let mut appender = + build_file_appender(&opts, LogFileKind::Default, Some(&retention)).unwrap(); retention.initialize(); appender.write_all(b"current").unwrap(); @@ -616,7 +626,8 @@ mod tests { ..Default::default() }; let retention = DirectoryRetention::new(directory.path(), ReadableSize(1), 0).unwrap(); - let mut appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention)); + let mut appender = + build_file_appender(&opts, LogFileKind::Default, Some(&retention)).unwrap(); retention.initialize(); appender.write_all(b"current").unwrap();