diff --git a/Cargo.lock b/Cargo.lock
index d8822ef5c38..c716ca8c81a 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2931,6 +2931,7 @@ dependencies = [
"backtrace",
"common-base",
"common-error",
+ "common-macro",
"common-version",
"console-subscriber",
"greptime-proto",
@@ -2945,6 +2946,7 @@ dependencies = [
"prometheus 0.14.0",
"serde",
"serde_json",
+ "snafu 0.8.6",
"tempfile",
"tokio",
"tracing",
diff --git a/config/config.md b/config/config.md
index 27aeabab52c..16e98f77aea 100644
--- a/config/config.md
+++ b/config/config.md
@@ -260,7 +260,7 @@
| `logging.enable_file_logging` | Bool | `true` | Whether to write logs to files in `dir`. |
| `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
| `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
-| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. |
+| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
Requires symbolic links in the log directory; initialization fails if they cannot be created.
On Windows, enable Developer Mode or leave this at `0B`. |
| `logging.enable_per_region_metrics` | Bool | `false` | Whether to enable per-region metrics.
Default to false. |
| `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. |
| `logging.otlp_headers` | -- | -- | Additional OTLP headers, only valid when using OTLP http |
@@ -422,7 +422,7 @@
| `logging.enable_file_logging` | Bool | `true` | Whether to write logs to files in `dir`. |
| `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
| `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
-| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. |
+| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
Requires symbolic links in the log directory; initialization fails if they cannot be created.
On Windows, enable Developer Mode or leave this at `0B`. |
| `logging.enable_per_region_metrics` | Bool | `false` | Whether to enable per-region metrics.
Default to false. |
| `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. |
| `logging.otlp_headers` | -- | -- | Additional OTLP headers, only valid when using OTLP http |
@@ -541,7 +541,7 @@
| `logging.enable_file_logging` | Bool | `true` | Whether to write logs to files in `dir`. |
| `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
| `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
-| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. |
+| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
Requires symbolic links in the log directory; initialization fails if they cannot be created.
On Windows, enable Developer Mode or leave this at `0B`. |
| `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. |
| `logging.otlp_headers` | -- | -- | Additional OTLP headers, only valid when using OTLP http |
| `logging.tracing_sample_ratio` | -- | Unset | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
@@ -748,7 +748,7 @@
| `logging.enable_file_logging` | Bool | `true` | Whether to write logs to files in `dir`. |
| `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
| `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
-| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. |
+| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
Requires symbolic links in the log directory; initialization fails if they cannot be created.
On Windows, enable Developer Mode or leave this at `0B`. |
| `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. |
| `logging.otlp_headers` | -- | -- | Additional OTLP headers, only valid when using OTLP http |
| `logging.tracing_sample_ratio` | -- | Unset | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
@@ -809,7 +809,7 @@
| `logging.enable_file_logging` | Bool | `true` | Whether to write logs to files in `dir`. |
| `logging.log_format` | String | `text` | The log format. Can be `text`/`json`. |
| `logging.max_log_files` | Integer | `720` | The maximum amount of log files. |
-| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable. |
+| `logging.max_log_dir_size` | String | `0B` | The maximum total size of managed log files in `dir`.
Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
Requires symbolic links in the log directory; initialization fails if they cannot be created.
On Windows, enable Developer Mode or leave this at `0B`. |
| `logging.otlp_export_protocol` | String | `http` | The OTLP tracing export protocol. Can be `grpc`/`http`. |
| `logging.otlp_headers` | -- | -- | Additional OTLP headers, only valid when using OTLP http |
| `logging.tracing_sample_ratio` | -- | Unset | The percentage of tracing will be sampled and exported.
Valid range `[0, 1]`, 1 means all traces are sampled, 0 means all traces are not sampled, the default value is 1.
ratio > 1 are treated as 1. Fractions < 0 are treated as 0 |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 048a3b7a39d..342ceb0a17a 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -906,6 +906,8 @@ max_log_files = 720
## The maximum total size of managed log files in `dir`.
## Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
+## Requires symbolic links in the log directory; initialization fails if they cannot be created.
+## On Windows, enable Developer Mode or leave this at `0B`.
max_log_dir_size = "0B"
## The OTLP tracing export protocol. Can be `grpc`/`http`.
diff --git a/config/flownode.example.toml b/config/flownode.example.toml
index fab31fed510..8faeb2cc227 100644
--- a/config/flownode.example.toml
+++ b/config/flownode.example.toml
@@ -126,6 +126,8 @@ max_log_files = 720
## The maximum total size of managed log files in `dir`.
## Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
+## Requires symbolic links in the log directory; initialization fails if they cannot be created.
+## On Windows, enable Developer Mode or leave this at `0B`.
max_log_dir_size = "0B"
## The OTLP tracing export protocol. Can be `grpc`/`http`.
diff --git a/config/frontend.example.toml b/config/frontend.example.toml
index b8a73240345..d5638b66f58 100644
--- a/config/frontend.example.toml
+++ b/config/frontend.example.toml
@@ -464,6 +464,8 @@ max_log_files = 720
## The maximum total size of managed log files in `dir`.
## Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
+## Requires symbolic links in the log directory; initialization fails if they cannot be created.
+## On Windows, enable Developer Mode or leave this at `0B`.
max_log_dir_size = "0B"
## Whether to enable per-region metrics.
diff --git a/config/metasrv.example.toml b/config/metasrv.example.toml
index fe4c38d02f9..35ba3ae0a98 100644
--- a/config/metasrv.example.toml
+++ b/config/metasrv.example.toml
@@ -392,6 +392,8 @@ max_log_files = 720
## The maximum total size of managed log files in `dir`.
## Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
+## Requires symbolic links in the log directory; initialization fails if they cannot be created.
+## On Windows, enable Developer Mode or leave this at `0B`.
max_log_dir_size = "0B"
## The OTLP tracing export protocol. Can be `grpc`/`http`.
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index 616b6f4f795..237f106992b 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -1110,6 +1110,8 @@ max_log_files = 720
## The maximum total size of managed log files in `dir`.
## Old closed log files are removed before writing when necessary. Active files may exceed it. Set to `0B` to disable.
+## Requires symbolic links in the log directory; initialization fails if they cannot be created.
+## On Windows, enable Developer Mode or leave this at `0B`.
max_log_dir_size = "0B"
## Whether to enable per-region metrics.
diff --git a/src/cmd/src/cli.rs b/src/cmd/src/cli.rs
index 3739d4cc4fc..92aca3a5625 100644
--- a/src/cmd/src/cli.rs
+++ b/src/cmd/src/cli.rs
@@ -77,7 +77,8 @@ impl Command {
&TracingOptions::default(),
None,
None,
- );
+ )
+ .context(crate::error::InitLoggingSnafu)?;
let tool = self.cmd.build().await.context(error::BuildCliSnafu)?;
let instance = Instance {
diff --git a/src/cmd/src/datanode/builder.rs b/src/cmd/src/datanode/builder.rs
index 6df7ac728e8..fc933a9c5be 100644
--- a/src/cmd/src/datanode/builder.rs
+++ b/src/cmd/src/datanode/builder.rs
@@ -67,7 +67,8 @@ impl InstanceBuilder {
&dn_opts.tracing,
dn_opts.node_id.map(|x| x.to_string()),
None,
- );
+ )
+ .context(crate::error::InitLoggingSnafu)?;
common_runtime::init_datanode_runtimes(&opts.runtime);
diff --git a/src/cmd/src/error.rs b/src/cmd/src/error.rs
index 7b0decf05ca..84553439ab1 100644
--- a/src/cmd/src/error.rs
+++ b/src/cmd/src/error.rs
@@ -23,6 +23,13 @@ use snafu::{Location, Snafu};
#[snafu(visibility(pub))]
#[stack_trace_debug]
pub enum Error {
+ #[snafu(display("Failed to initialize logging"))]
+ InitLogging {
+ #[snafu(implicit)]
+ location: Location,
+ source: common_telemetry::logging::InitLoggingError,
+ },
+
#[snafu(display("Failed to install ring crypto provider: {}", msg))]
InitTlsProvider {
#[snafu(implicit)]
@@ -337,6 +344,7 @@ pub type Result = std::result::Result;
impl ErrorExt for Error {
fn status_code(&self) -> StatusCode {
match self {
+ Error::InitLogging { source, .. } => source.status_code(),
Error::StartDatanode { source, .. } => source.status_code(),
Error::StartFrontend { source, .. } => source.status_code(),
Error::ShutdownDatanode { source, .. } => source.status_code(),
diff --git a/src/cmd/src/flownode.rs b/src/cmd/src/flownode.rs
index 0dfcf85e2e7..91faf313dc3 100644
--- a/src/cmd/src/flownode.rs
+++ b/src/cmd/src/flownode.rs
@@ -251,7 +251,8 @@ impl StartCommand {
&opts.component.tracing,
opts.component.node_id.map(|x| x.to_string()),
None,
- );
+ )
+ .context(crate::error::InitLoggingSnafu)?;
common_runtime::init_global_runtimes(&opts.runtime);
diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs
index 4348d55799d..7a430d7619d 100644
--- a/src/cmd/src/frontend.rs
+++ b/src/cmd/src/frontend.rs
@@ -384,7 +384,8 @@ impl StartCommand {
&opts.component.tracing,
opts.component.node_id.clone(),
Some(&opts.component.slow_query),
- );
+ )
+ .context(crate::error::InitLoggingSnafu)?;
common_runtime::init_global_runtimes(&opts.runtime);
diff --git a/src/cmd/src/metasrv.rs b/src/cmd/src/metasrv.rs
index 369f5c428d1..7df4b92ab00 100644
--- a/src/cmd/src/metasrv.rs
+++ b/src/cmd/src/metasrv.rs
@@ -324,7 +324,8 @@ impl StartCommand {
&opts.component.tracing,
None,
None,
- );
+ )
+ .context(crate::error::InitLoggingSnafu)?;
common_runtime::init_global_runtimes(&opts.runtime);
diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
index 9dbcb89abc4..934429517f4 100644
--- a/src/cmd/src/standalone.rs
+++ b/src/cmd/src/standalone.rs
@@ -498,7 +498,8 @@ impl StartCommand {
&opts.component.tracing,
None,
Some(&opts.component.slow_query),
- );
+ )
+ .context(crate::error::InitLoggingSnafu)?;
common_runtime::init_standalone_runtimes(&opts.runtime);
diff --git a/src/common/telemetry/Cargo.toml b/src/common/telemetry/Cargo.toml
index 1e146a5a75e..33be63ed92e 100644
--- a/src/common/telemetry/Cargo.toml
+++ b/src/common/telemetry/Cargo.toml
@@ -15,6 +15,7 @@ workspace = true
backtrace = "0.3"
common-base.workspace = true
common-error.workspace = true
+common-macro.workspace = true
common-version.workspace = true
console-subscriber = { version = "0.1", optional = true }
greptime-proto.workspace = true
@@ -31,6 +32,7 @@ parking_lot.workspace = true
prometheus.workspace = true
serde.workspace = true
serde_json.workspace = true
+snafu.workspace = true
tokio.workspace = true
tracing = "0.1"
tracing-appender.workspace = true
diff --git a/src/common/telemetry/src/logging.rs b/src/common/telemetry/src/logging.rs
index 491b84f3fab..5154f7dc67e 100644
--- a/src/common/telemetry/src/logging.rs
+++ b/src/common/telemetry/src/logging.rs
@@ -15,6 +15,7 @@
//! logging stuffs, inspired by databend
mod file_retention;
+use std::any::Any;
use std::collections::HashMap;
use std::env;
use std::io::IsTerminal;
@@ -24,6 +25,9 @@ use std::time::Duration;
use common_base::readable_size::ReadableSize;
use common_base::serde::empty_string_as_default;
+use common_error::ext::ErrorExt;
+use common_error::status_code::StatusCode;
+use common_macro::stack_trace_debug;
use file_retention::{DirectoryRetention, LogFileKind, build_file_appender};
use once_cell::sync::{Lazy, OnceCell};
use opentelemetry::trace::TracerProvider;
@@ -33,6 +37,7 @@ use opentelemetry_sdk::propagation::TraceContextPropagator;
use opentelemetry_sdk::trace::{Sampler, Tracer};
use opentelemetry_semantic_conventions::resource;
use serde::{Deserialize, Serialize};
+use snafu::{Location, Snafu};
use tracing::callsite;
use tracing::metadata::LevelFilter;
use tracing_appender::non_blocking::{NonBlocking, WorkerGuard};
@@ -45,6 +50,31 @@ use tracing_subscriber::{EnvFilter, Registry, filter};
use crate::tracing_sampler::{TracingSampleOptions, create_sampler};
+/// Failure to initialize a file appender.
+#[derive(Snafu)]
+#[stack_trace_debug]
+pub enum InitLoggingError {
+ #[snafu(display("Failed to initialize file logging in {directory}: {error}.{hint}"))]
+ FileAppender {
+ directory: String,
+ #[snafu(source)]
+ error: tracing_appender::rolling::InitError,
+ hint: &'static str,
+ #[snafu(implicit)]
+ location: Location,
+ },
+}
+
+impl ErrorExt for InitLoggingError {
+ fn status_code(&self) -> StatusCode {
+ StatusCode::InvalidArguments
+ }
+
+ fn as_any(&self) -> &dyn Any {
+ self
+ }
+}
+
/// The default endpoint when use gRPC exporter protocol.
pub const DEFAULT_OTLP_GRPC_ENDPOINT: &str = "http://localhost:4317";
@@ -474,13 +504,16 @@ pub fn init_default_ut_logging() {
level: Some(level),
..Default::default()
};
- *g = Some(init_global_logging(
- "unittest",
- &opts,
- &TracingOptions::default(),
- None,
- None,
- ));
+ *g = Some(
+ init_global_logging(
+ "unittest",
+ &opts,
+ &TracingOptions::default(),
+ None,
+ None,
+ )
+ .expect("failed to initialize test logging"),
+ );
crate::info!("logs dir = {}", dir);
});
@@ -498,15 +531,12 @@ pub fn init_global_logging(
tracing_opts: &TracingOptions,
node_id: Option,
slow_query_opts: Option<&SlowQueryOptions>,
-) -> Vec {
- static START: Once = Once::new();
+) -> Result, InitLoggingError> {
+ static START: OnceCell<()> = OnceCell::new();
let mut guards = vec![];
let node_id = node_id.unwrap_or_else(|| "none".to_string());
- START.call_once(|| {
- // Enable log compatible layer to convert log record to tracing span.
- LogTracer::init().expect("log tracer must be valid");
-
+ START.get_or_try_init(|| -> Result<(), InitLoggingError> {
let stdout_writer = if opts.append_stdout {
let (writer, guard) = tracing_appender::non_blocking(std::io::stdout());
guards.push(guard);
@@ -531,7 +561,7 @@ pub fn init_global_logging(
// Configure the file logging layer with rolling policy.
let file_logging_layer = if file_logging_enabled {
let rolling_appender =
- build_file_appender(opts, LogFileKind::Default, retention.as_ref());
+ build_file_appender(opts, LogFileKind::Default, retention.as_ref())?;
let (writer, guard) = tracing_appender::non_blocking(rolling_appender);
guards.push(guard);
@@ -543,7 +573,7 @@ pub fn init_global_logging(
// Configure the error file logging layer with rolling policy.
let err_file_logging_layer = if file_logging_enabled {
let rolling_appender =
- build_file_appender(opts, LogFileKind::Error, retention.as_ref());
+ build_file_appender(opts, LogFileKind::Error, retention.as_ref())?;
let (writer, guard) = tracing_appender::non_blocking(rolling_appender);
guards.push(guard);
@@ -555,14 +585,23 @@ pub fn init_global_logging(
None
};
- if slow_query_logging_enabled {
+ let slow_query_logger = if slow_query_logging_enabled {
+ Some(build_slow_query_logger(
+ opts,
+ stdout_writer,
+ retention.as_ref(),
+ &mut guards,
+ )?)
+ } else {
+ None
+ };
+
+ // Build all file appenders before installing any global logging state.
+ // A filesystem failure can then be returned without poisoning initialization.
+ LogTracer::init().expect("log tracer must be valid");
+ if let Some(logger) = slow_query_logger {
SLOW_QUERY_LOGGER
- .set(build_slow_query_logger(
- opts,
- stdout_writer,
- retention.as_ref(),
- &mut guards,
- ))
+ .set(logger)
.expect("slow query logger already initialized");
}
@@ -659,9 +698,10 @@ pub fn init_global_logging(
tracing::subscriber::set_global_default(subscriber)
.expect("error setting global tracing subscriber");
- });
+ Ok(())
+ })?;
- guards
+ Ok(guards)
}
fn create_tracer(app_name: &str, node_id: &str, opts: &LoggingOptions) -> Tracer {
@@ -747,12 +787,12 @@ fn build_slow_query_logger(
stdout_writer: Option,
retention: Option<&DirectoryRetention>,
guards: &mut Vec,
-) -> tracing::Dispatch {
+) -> Result {
let stdout_layer = stdout_writer.map(|writer| {
build_logging_layer(writer, opts.log_format, std::io::stdout().is_terminal())
});
let file_layer = if opts.enable_file_logging && !opts.dir.is_empty() {
- let rolling_appender = build_file_appender(opts, LogFileKind::SlowQuery, retention);
+ let rolling_appender = build_file_appender(opts, LogFileKind::SlowQuery, retention)?;
let (writer, guard) = tracing_appender::non_blocking(rolling_appender);
guards.push(guard);
Some(build_logging_layer(writer, opts.log_format, false))
@@ -764,12 +804,12 @@ fn build_slow_query_logger(
*metadata.level() == tracing::Level::INFO && metadata.fields().field("slow").is_some()
})
.with_max_level_hint(LevelFilter::INFO);
- tracing::Dispatch::new(
+ Ok(tracing::Dispatch::new(
Registry::default()
.with(filter)
.with(stdout_layer)
.with(file_layer),
- )
+ ))
}
fn build_logging_layer(
@@ -802,6 +842,100 @@ mod tests {
use super::*;
+ #[test]
+ fn test_file_logging_initialization_failure() {
+ const CHILD_ENV: &str = "GREPTIME_TEST_LOGGING_INIT_FAILURE";
+ if env::var_os(CHILD_ENV).is_none() {
+ let output = std::process::Command::new(env::current_exe().unwrap())
+ .args([
+ "--exact",
+ "logging::tests::test_file_logging_initialization_failure",
+ "--nocapture",
+ ])
+ .env(CHILD_ENV, "1")
+ .output()
+ .unwrap();
+ assert!(
+ output.status.success(),
+ "{}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ return;
+ }
+
+ let dir = tempfile::tempdir().unwrap();
+ let mut opts = LoggingOptions {
+ dir: dir.path().to_str().unwrap().to_string(),
+ max_log_dir_size: ReadableSize::mb(1),
+ append_stdout: false,
+ ..Default::default()
+ };
+ let slow_opts = SlowQueryOptions {
+ enable: true,
+ record_type: SlowQueriesRecordType::Log,
+ ..Default::default()
+ };
+ for link in [
+ ".greptimedb.current",
+ ".greptimedb-err.current",
+ ".greptimedb-slow-queries.current",
+ ] {
+ let path = dir.path().join(link);
+ // A directory collision fails even on runners allowed to create symlinks.
+ std::fs::create_dir(&path).unwrap();
+ let error = init_global_logging(
+ "test",
+ &opts,
+ &TracingOptions::default(),
+ None,
+ Some(&slow_opts),
+ )
+ .unwrap_err();
+ let message = error.to_string();
+ assert!(message.contains("logging.max_log_dir_size"), "{message}");
+ assert!(message.contains(&opts.dir), "{message}");
+ assert!(message.contains("Developer Mode"), "{message}");
+ assert!(LOG_RELOAD_HANDLE.get().is_none());
+ assert!(SLOW_QUERY_LOGGER.get().is_none());
+ std::fs::remove_dir(path).unwrap();
+ for entry in std::fs::read_dir(dir.path()).unwrap() {
+ let entry = entry.unwrap();
+ if entry.file_type().unwrap().is_symlink() {
+ std::fs::remove_file(entry.path()).unwrap();
+ }
+ }
+ }
+
+ // Disabling retention succeeds with the same collision and after failed attempts.
+ std::fs::create_dir(dir.path().join(".greptimedb.current")).unwrap();
+ opts.max_log_dir_size = ReadableSize::default();
+ let guards = init_global_logging(
+ "test",
+ &opts,
+ &TracingOptions::default(),
+ None,
+ Some(&slow_opts),
+ )
+ .unwrap();
+ crate::error!("logging works after initialization failures");
+ drop(guards);
+ let error_log = std::fs::read_dir(dir.path())
+ .unwrap()
+ .map(|entry| entry.unwrap().path())
+ .find(|path| {
+ path.file_name()
+ .unwrap()
+ .to_string_lossy()
+ .starts_with("greptimedb-err.")
+ })
+ .unwrap();
+ assert!(
+ std::fs::read_to_string(error_log)
+ .unwrap()
+ .contains("logging works after initialization failures")
+ );
+ }
+
#[test]
fn test_slow_query_logging() {
const CASE_ENV: &str = "GREPTIME_TEST_SLOW_QUERY_LOGGING";
@@ -915,7 +1049,8 @@ mod tests {
&TracingOptions::default(),
None,
(case != "no_options").then_some(&slow_opts),
- );
+ )
+ .unwrap();
for level in ["warn", "error", "off", "info"] {
LOG_RELOAD_HANDLE
diff --git a/src/common/telemetry/src/logging/file_retention.rs b/src/common/telemetry/src/logging/file_retention.rs
index 0a79468851f..d2a962ad748 100644
--- a/src/common/telemetry/src/logging/file_retention.rs
+++ b/src/common/telemetry/src/logging/file_retention.rs
@@ -24,9 +24,10 @@ use std::time::SystemTime;
use common_base::readable_size::ReadableSize;
use parking_lot::Mutex;
+use snafu::ResultExt;
use tracing_appender::rolling::{RollingFileAppender, Rotation};
-use crate::logging::LoggingOptions;
+use crate::logging::{FileAppenderSnafu, InitLoggingError, LoggingOptions};
/// A managed file-log kind.
#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)]
@@ -421,7 +422,7 @@ pub(crate) fn build_file_appender(
opts: &LoggingOptions,
kind: LogFileKind,
retention: Option<&DirectoryRetention>,
-) -> Box {
+) -> Result, InitLoggingError> {
// Directory retention owns size and count pruning so its index remains authoritative.
let upstream_max_log_files = if retention.is_some() {
0
@@ -440,18 +441,24 @@ pub(crate) fn build_file_appender(
// before creating it, so a missing directory prints a misleading error. If this fails,
// `build` reports it.
let _ = fs::create_dir_all(&opts.dir);
- let appender = builder.build(&opts.dir).unwrap_or_else(|error| {
- panic!(
- "initializing rolling file appender at {} failed: {}",
- opts.dir, error
- )
- });
+ let appender = builder.build(&opts.dir).with_context(|_| FileAppenderSnafu {
+ directory: opts.dir.clone(),
+ hint: if retention.is_some() {
+ " logging.max_log_dir_size requires creating symbolic links in this directory. Check directory permissions and conflicting paths; on Windows, enable Developer Mode or set logging.max_log_dir_size = \"0B\" to disable size-based retention."
+ } else {
+ ""
+ },
+ })?;
if let Some(retention) = retention {
retention.register(kind);
- Box::new(RetentionAppender::new(appender, retention.clone(), kind))
+ Ok(Box::new(RetentionAppender::new(
+ appender,
+ retention.clone(),
+ kind,
+ )))
} else {
- Box::new(appender)
+ Ok(Box::new(appender))
}
}
@@ -521,7 +528,7 @@ mod tests {
};
let retention = DirectoryRetention::new(directory.path(), ReadableSize::mb(1), 0).unwrap();
- let _appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention));
+ let _appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention)).unwrap();
let symlink = directory.path().join(".greptimedb.current");
assert!(symlink.is_symlink());
@@ -542,7 +549,8 @@ mod tests {
..Default::default()
};
let retention = DirectoryRetention::new(directory.path(), ReadableSize(1024), 0).unwrap();
- let mut appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention));
+ let mut appender =
+ build_file_appender(&opts, LogFileKind::Default, Some(&retention)).unwrap();
retention.initialize();
appender.write_all(b"current").unwrap();
@@ -570,7 +578,8 @@ mod tests {
..Default::default()
};
let retention = DirectoryRetention::new(directory.path(), ReadableSize(1024), 0).unwrap();
- let mut appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention));
+ let mut appender =
+ build_file_appender(&opts, LogFileKind::Default, Some(&retention)).unwrap();
retention.initialize();
appender.write_all(b"current").unwrap();
@@ -593,7 +602,8 @@ mod tests {
..Default::default()
};
let retention = DirectoryRetention::new(directory.path(), ReadableSize::gb(1), 2).unwrap();
- let mut appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention));
+ let mut appender =
+ build_file_appender(&opts, LogFileKind::Default, Some(&retention)).unwrap();
retention.initialize();
appender.write_all(b"current").unwrap();
@@ -616,7 +626,8 @@ mod tests {
..Default::default()
};
let retention = DirectoryRetention::new(directory.path(), ReadableSize(1), 0).unwrap();
- let mut appender = build_file_appender(&opts, LogFileKind::Default, Some(&retention));
+ let mut appender =
+ build_file_appender(&opts, LogFileKind::Default, Some(&retention)).unwrap();
retention.initialize();
appender.write_all(b"current").unwrap();