diff --git a/config/config.md b/config/config.md
index d450609a926..e370bb157a0 100644
--- a/config/config.md
+++ b/config/config.md
@@ -174,7 +174,7 @@
| `storage.http_client.timeout` | String | `30s` | The total request timeout, applied from when the request starts connecting until the response body has finished.
Also considered a total deadline. |
| `storage.http_client.pool_idle_timeout` | String | `90s` | The timeout for idle sockets being kept-alive. |
| `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification
**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. |
-| `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines.
Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail. |
+| `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines.
Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail.
The default configuration lists `mito` and `file`; `metric` is always registered, so naming
it only supplies its options rather than enabling the engine. |
| `region_engine.mito` | -- | -- | The Mito engine options. |
| `region_engine.mito.num_workers` | Integer | Auto | Number of region workers. Defaults to half of the available CPU cores, rounded up. |
| `region_engine.mito.worker_channel_size` | Integer | `128` | Request channel size of each worker. |
@@ -650,7 +650,7 @@
| `storage.http_client.timeout` | String | `30s` | The total request timeout, applied from when the request starts connecting until the response body has finished.
Also considered a total deadline. |
| `storage.http_client.pool_idle_timeout` | String | `90s` | The timeout for idle sockets being kept-alive. |
| `storage.http_client.skip_ssl_validation` | Bool | `false` | To skip the ssl verification
**Security Notice**: Setting `skip_ssl_validation = true` disables certificate verification, making connections vulnerable to man-in-the-middle attacks. Only use this in development or trusted private networks. |
-| `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines.
Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail. |
+| `[[region_engine]]` | -- | -- | The region engine options. You can configure multiple region engines.
Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail.
The default configuration lists `mito` and `file`; `metric` is always registered, so naming
it only supplies its options rather than enabling the engine. |
| `region_engine.mito` | -- | -- | The Mito engine options. |
| `region_engine.mito.num_workers` | Integer | Auto | Number of region workers. Defaults to half of the available CPU cores, rounded up. |
| `region_engine.mito.worker_channel_size` | Integer | `128` | Request channel size of each worker. |
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
index 4b8eb41242e..bfa3afc0bcf 100644
--- a/config/datanode.example.toml
+++ b/config/datanode.example.toml
@@ -509,6 +509,8 @@ skip_ssl_validation = false
## The region engine options. You can configure multiple region engines.
## Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail.
+## The default configuration lists `mito` and `file`; `metric` is always registered, so naming
+## it only supplies its options rather than enabling the engine.
[[region_engine]]
## The Mito engine options.
diff --git a/config/standalone.example.toml b/config/standalone.example.toml
index f70277aa599..f73a3f8a325 100644
--- a/config/standalone.example.toml
+++ b/config/standalone.example.toml
@@ -729,6 +729,8 @@ skip_ssl_validation = false
## The region engine options. You can configure multiple region engines.
## Each engine type (mito, file, metric) may appear only once; duplicates cause startup to fail.
+## The default configuration lists `mito` and `file`; `metric` is always registered, so naming
+## it only supplies its options rather than enabling the engine.
[[region_engine]]
## The Mito engine options.
diff --git a/src/cmd/src/options.rs b/src/cmd/src/options.rs
index e09b3c8f733..ed88431e4fd 100644
--- a/src/cmd/src/options.rs
+++ b/src/cmd/src/options.rs
@@ -65,6 +65,10 @@ where
fn env_list_keys() -> Option<&'static [&'static str]> {
T::env_list_keys()
}
+
+ fn env_tagged_seq_keys() -> Option<&'static [&'static str]> {
+ T::env_tagged_seq_keys()
+ }
}
/// A serde `deserialize_with` helper that parses the `plugins` field leniently.
diff --git a/src/cmd/tests/load_config_test.rs b/src/cmd/tests/load_config_test.rs
index 2781183c0c8..2a41c120b23 100644
--- a/src/cmd/tests/load_config_test.rs
+++ b/src/cmd/tests/load_config_test.rs
@@ -902,3 +902,216 @@ fn test_load_metric_config_with_removed_sparse_primary_key_encoding() {
Duration::from_secs(30)
);
}
+
+/// Returns the mito engine config of a loaded `region_engine` sequence.
+fn find_mito(region_engine: &[RegionEngineConfig]) -> &MitoConfig {
+ region_engine
+ .iter()
+ .find_map(|engine| match engine {
+ RegionEngineConfig::Mito(config) => Some(config),
+ _ => None,
+ })
+ .expect("the mito engine config")
+}
+
+#[test]
+fn test_load_region_engine_mito_field_from_env() {
+ let env_prefix = "REGION_ENGINE_MITO_FIELD_UT";
+ let env_key = [
+ env_prefix,
+ "REGION_ENGINE",
+ "MITO",
+ "GLOBAL_WRITE_BUFFER_REJECT_SIZE",
+ ]
+ .join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("4GB"), || {
+ let options =
+ GreptimeOptions::::load_layered_options(None, env_prefix).unwrap();
+
+ // The addressed field wins, the fields next to it keep their default value.
+ let mito = find_mito(&options.component.region_engine);
+ assert_eq!(mito.global_write_buffer_reject_size, ReadableSize::gb(4));
+ assert_eq!(
+ mito.global_write_buffer_size,
+ MitoConfig::default().global_write_buffer_size
+ );
+ assert_eq!(mito.index, MitoConfig::default().index);
+
+ // The other entries of the default sequence are intact.
+ assert!(
+ options
+ .component
+ .region_engine
+ .iter()
+ .any(|engine| matches!(engine, RegionEngineConfig::File(_))),
+ "the file engine config should survive an env override of the mito engine"
+ );
+ });
+}
+
+#[test]
+fn test_load_region_engine_mito_field_from_env_standalone() {
+ let env_prefix = "REGION_ENGINE_MITO_FIELD_STANDALONE_UT";
+ let env_key = [
+ env_prefix,
+ "REGION_ENGINE",
+ "MITO",
+ "GLOBAL_WRITE_BUFFER_REJECT_SIZE",
+ ]
+ .join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("4GB"), || {
+ let options =
+ GreptimeOptions::::load_layered_options(None, env_prefix).unwrap();
+
+ let mito = find_mito(&options.component.region_engine);
+ assert_eq!(mito.global_write_buffer_reject_size, ReadableSize::gb(4));
+ assert_eq!(
+ mito.global_write_buffer_size,
+ MitoConfig::default().global_write_buffer_size
+ );
+ assert!(
+ options
+ .component
+ .region_engine
+ .iter()
+ .any(|engine| matches!(engine, RegionEngineConfig::File(_))),
+ "the file engine config should survive an env override of the mito engine"
+ );
+ });
+}
+
+#[test]
+fn test_load_region_engine_metric_entry_from_env() {
+ let env_prefix = "REGION_ENGINE_METRIC_FIELD_UT";
+ let env_key = [
+ env_prefix,
+ "REGION_ENGINE",
+ "METRIC",
+ "FLUSH_METADATA_REGION_INTERVAL",
+ ]
+ .join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("1m"), || {
+ let options =
+ GreptimeOptions::::load_layered_options(None, env_prefix).unwrap();
+
+ // `metric` is not in the default sequence, so it is appended instead of replacing
+ // the whole sequence.
+ assert_eq!(
+ options.component.region_engine.len(),
+ DatanodeOptions::default().region_engine.len() + 1
+ );
+ let metric = options
+ .component
+ .region_engine
+ .iter()
+ .find_map(|engine| match engine {
+ RegionEngineConfig::Metric(config) => Some(config),
+ _ => None,
+ })
+ .expect("the metric engine config");
+ assert_eq!(
+ metric.flush_metadata_region_interval,
+ Duration::from_secs(60)
+ );
+
+ // The engines of the default sequence keep their default options.
+ let mito = find_mito(&options.component.region_engine);
+ assert_eq!(
+ mito.global_write_buffer_reject_size,
+ MitoConfig::default().global_write_buffer_reject_size
+ );
+ assert!(
+ options
+ .component
+ .region_engine
+ .iter()
+ .any(|engine| matches!(engine, RegionEngineConfig::File(_)))
+ );
+ });
+}
+
+#[test]
+fn test_load_region_engine_env_patch_is_replaced_by_config_file() {
+ let config = tempfile::NamedTempFile::new().unwrap();
+ std::fs::write(
+ config.path(),
+ "[[region_engine]]\n[region_engine.metric]\nflush_metadata_region_interval = \"2m\"\n",
+ )
+ .unwrap();
+
+ let env_prefix = "REGION_ENGINE_FILE_WINS_UT";
+ let env_key = [
+ env_prefix,
+ "REGION_ENGINE",
+ "MITO",
+ "GLOBAL_WRITE_BUFFER_REJECT_SIZE",
+ ]
+ .join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("4GB"), || {
+ let options = GreptimeOptions::::load_layered_options(
+ config.path().to_str(),
+ env_prefix,
+ )
+ .unwrap();
+
+ // A sequence is a single value: the config file replaces the whole value produced by
+ // the environment, so the environment patch (which is merged onto the default
+ // entries) is dropped. That keeps the precedence file > env > default of every other
+ // option, and means a config file has to spell out every engine it wants.
+ assert_eq!(
+ options.component.region_engine,
+ vec![RegionEngineConfig::Metric(MetricEngineConfig {
+ flush_metadata_region_interval: Duration::from_secs(120),
+ })]
+ );
+ });
+}
+
+#[test]
+fn test_load_region_engine_env_patch_with_config_file_lacking_region_engine() {
+ // The usual deployment shape: a config file that sets other options, plus an environment
+ // variable patching one region engine. The file says nothing about `region_engine`, so
+ // there is nothing that could replace the patched sequence.
+ let config = tempfile::NamedTempFile::new().unwrap();
+ std::fs::write(config.path(), "node_id = 7\n").unwrap();
+
+ let env_prefix = "REGION_ENGINE_FILE_NO_ENGINE_UT";
+ let env_key = [
+ env_prefix,
+ "REGION_ENGINE",
+ "MITO",
+ "GLOBAL_WRITE_BUFFER_REJECT_SIZE",
+ ]
+ .join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("4GB"), || {
+ let options = GreptimeOptions::::load_layered_options(
+ config.path().to_str(),
+ env_prefix,
+ )
+ .unwrap();
+
+ // The option of the file is applied...
+ assert_eq!(options.component.node_id, Some(7));
+ // ...the environment patch applies on top of the default engine entries...
+ let mito = find_mito(&options.component.region_engine);
+ assert_eq!(mito.global_write_buffer_reject_size, ReadableSize::gb(4));
+ assert_eq!(
+ mito.global_write_buffer_size,
+ MitoConfig::default().global_write_buffer_size
+ );
+ // ...and the rest of the default sequence is still configured.
+ assert!(
+ options
+ .component
+ .region_engine
+ .iter()
+ .any(|engine| matches!(engine, RegionEngineConfig::File(_))),
+ "the file engine entry should survive"
+ );
+ });
+}
diff --git a/src/common/config/src/config.rs b/src/common/config/src/config.rs
index 85ce3d206f6..8dfd195d0bc 100644
--- a/src/common/config/src/config.rs
+++ b/src/common/config/src/config.rs
@@ -17,7 +17,10 @@ use serde::Serialize;
use serde::de::DeserializeOwned;
use snafu::ResultExt;
-use crate::error::{LoadLayeredConfigSnafu, Result, SerdeJsonSnafu, TomlFormatSnafu};
+use crate::error::{
+ LoadLayeredConfigSnafu, Result, SerdeJsonSnafu, TaggedSeqNotTopLevelSnafu, TomlFormatSnafu,
+};
+use crate::tagged_seq::TaggedSeqEnvSource;
/// Separator for environment variables. For example, `DATANODE__STORAGE__MANIFEST__CHECKPOINT_MARGIN`.
pub const ENV_VAR_SEP: &str = "__";
@@ -64,9 +67,27 @@ pub trait Configurable: Serialize + DeserializeOwned + Default + Sized {
let default_config = File::from_str(&json_str, FileFormat::Json);
// Add default values and environment variables as the sources of the configuration.
- let mut layered_config = config::Config::builder()
- .add_source(default_config)
- .add_source(env_source);
+ let mut layered_config = config::Config::builder().add_source(default_config);
+
+ // A sequence of tagged tables cannot be addressed by plain dotted env keys: the path
+ // walk of `config` drops the default sequence. Such keys are merged onto the default
+ // entries by tag name instead.
+ layered_config = match Self::env_tagged_seq_keys() {
+ Some(keys) if !keys.is_empty() => {
+ // The default entries are looked up by literal key, so a dotted key would find
+ // none of them. Reject it instead of silently dropping the defaults.
+ if let Some(key) = keys.iter().find(|key| key.contains('.')) {
+ return TaggedSeqNotTopLevelSnafu {
+ key: (*key).to_string(),
+ }
+ .fail();
+ }
+
+ let defaults = serde_json::to_value(&default_opts).context(SerdeJsonSnafu)?;
+ layered_config.add_source(TaggedSeqEnvSource::new(env_source, keys, &defaults))
+ }
+ _ => layered_config.add_source(env_source),
+ };
// Add config file as the source of the configuration if it is specified.
if let Some(config_file) = config_file {
@@ -93,6 +114,22 @@ pub trait Configurable: Serialize + DeserializeOwned + Default + Sized {
None
}
+ /// Keys whose value is a sequence of single-key tagged tables (e.g. `region_engine`,
+ /// serialized from `Vec` with externally tagged variants). Environment variables
+ /// address them by tag name (`KEY____`) instead of numeric index, and the
+ /// overrides are merged onto the corresponding default entries.
+ ///
+ /// Keys must name a **top-level** field of the options struct: the default entries to
+ /// merge onto are looked up by literal key in the serialized defaults, so a dotted key is
+ /// rejected instead of silently leaving the defaults out of the merged sequence.
+ ///
+ /// A configuration file that configures such a key as a whole replaces the merged value,
+ /// so the environment variables for it are ineffective then and the file has to spell out
+ /// every entry it wants.
+ fn env_tagged_seq_keys() -> Option<&'static [&'static str]> {
+ None
+ }
+
/// Serialize the configuration to a TOML string.
fn to_toml(&self) -> Result {
toml::to_string(&self).context(TomlFormatSnafu)
@@ -102,7 +139,9 @@ pub trait Configurable: Serialize + DeserializeOwned + Default + Sized {
#[cfg(test)]
mod tests {
use std::io::Write;
+ use std::time::Duration;
+ use common_base::readable_size::ReadableSize;
use common_telemetry::logging::LoggingOptions;
use common_test_util::temp_dir::create_named_temp_file;
use common_wal::config::DatanodeWalConfig;
@@ -264,4 +303,358 @@ mod tests {
},
);
}
+
+ /// A sequence of single-key tagged tables, the shape of `Vec` with externally
+ /// tagged variants.
+ #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+ #[serde(default)]
+ struct TestIndexConfig {
+ aux_path: String,
+ cache_percent: u8,
+ }
+
+ impl Default for TestIndexConfig {
+ fn default() -> Self {
+ Self {
+ aux_path: "/default/aux".to_string(),
+ cache_percent: 30,
+ }
+ }
+ }
+
+ #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+ #[serde(default)]
+ struct TestMitoConfig {
+ global_write_buffer_size: ReadableSize,
+ global_write_buffer_reject_size: ReadableSize,
+ index: TestIndexConfig,
+ }
+
+ impl Default for TestMitoConfig {
+ fn default() -> Self {
+ Self {
+ global_write_buffer_size: ReadableSize::mb(64),
+ global_write_buffer_reject_size: ReadableSize::gb(2),
+ index: TestIndexConfig::default(),
+ }
+ }
+ }
+
+ #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+ #[serde(default)]
+ struct TestFileConfig {
+ data_dir: String,
+ }
+
+ impl Default for TestFileConfig {
+ fn default() -> Self {
+ Self {
+ data_dir: "/default/data".to_string(),
+ }
+ }
+ }
+
+ #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+ #[serde(default)]
+ struct TestMetricConfig {
+ #[serde(with = "humantime_serde")]
+ flush_metadata_region_interval: Duration,
+ }
+
+ impl Default for TestMetricConfig {
+ fn default() -> Self {
+ Self {
+ flush_metadata_region_interval: Duration::from_secs(30),
+ }
+ }
+ }
+
+ #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+ #[serde(rename_all = "snake_case")]
+ enum TestRegionEngine {
+ Mito(TestMitoConfig),
+ File(TestFileConfig),
+ Metric(TestMetricConfig),
+ }
+
+ #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+ struct TestTaggedSeqConfig {
+ region_engine: Vec,
+ }
+
+ impl Default for TestTaggedSeqConfig {
+ fn default() -> Self {
+ Self {
+ region_engine: vec![
+ TestRegionEngine::Mito(TestMitoConfig::default()),
+ TestRegionEngine::File(TestFileConfig::default()),
+ ],
+ }
+ }
+ }
+
+ impl Configurable for TestTaggedSeqConfig {
+ fn env_tagged_seq_keys() -> Option<&'static [&'static str]> {
+ Some(&["region_engine"])
+ }
+ }
+
+ /// A config whose tagged sequence key is not a top-level field.
+ #[derive(Debug, Serialize, Deserialize, Default)]
+ struct TestDottedKeyConfig {
+ region_engine: Vec,
+ }
+
+ impl Configurable for TestDottedKeyConfig {
+ fn env_tagged_seq_keys() -> Option<&'static [&'static str]> {
+ Some(&["region_engine.mito"])
+ }
+ }
+
+ fn test_mito(config: &TestTaggedSeqConfig) -> &TestMitoConfig {
+ config
+ .region_engine
+ .iter()
+ .find_map(|engine| match engine {
+ TestRegionEngine::Mito(config) => Some(config),
+ _ => None,
+ })
+ .expect("the mito entry")
+ }
+
+ fn test_metric(config: &TestTaggedSeqConfig) -> &TestMetricConfig {
+ config
+ .region_engine
+ .iter()
+ .find_map(|engine| match engine {
+ TestRegionEngine::Metric(config) => Some(config),
+ _ => None,
+ })
+ .expect("the metric entry")
+ }
+
+ /// Loads with the given prefix and returns the message of the error, including the
+ /// source of the layered loading error.
+ fn load_error_text(env_prefix: &str) -> String {
+ let err = TestTaggedSeqConfig::load_layered_options(None, env_prefix).unwrap_err();
+ let mut text = err.to_string();
+ let mut source = std::error::Error::source(&err);
+ while let Some(error) = source {
+ text.push_str(&format!(": {error}"));
+ source = std::error::Error::source(error);
+ }
+ text
+ }
+
+ #[test]
+ fn test_tagged_seq_env_overrides_a_field() {
+ let env_prefix = "TAGGED_SEQ_FIELD_UT";
+ let env_key = [
+ env_prefix,
+ "REGION_ENGINE",
+ "MITO",
+ "GLOBAL_WRITE_BUFFER_REJECT_SIZE",
+ ]
+ .join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("4GB"), || {
+ let opts = TestTaggedSeqConfig::load_layered_options(None, env_prefix).unwrap();
+ let defaults = TestTaggedSeqConfig::default();
+
+ // The entries of the sequence that the environment does not mention are intact.
+ assert_eq!(opts.region_engine.len(), defaults.region_engine.len());
+ assert_eq!(opts.region_engine[1], defaults.region_engine[1]);
+
+ // The addressed field wins, the fields next to it keep their default value.
+ let mito = test_mito(&opts);
+ assert_eq!(mito.global_write_buffer_reject_size, ReadableSize::gb(4));
+ assert_eq!(
+ mito.global_write_buffer_size,
+ TestMitoConfig::default().global_write_buffer_size
+ );
+ assert_eq!(mito.index, TestIndexConfig::default());
+ });
+ }
+
+ #[test]
+ fn test_tagged_seq_env_appends_an_entry() {
+ let env_prefix = "TAGGED_SEQ_APPEND_UT";
+ let env_key = [
+ env_prefix,
+ "REGION_ENGINE",
+ "METRIC",
+ "FLUSH_METADATA_REGION_INTERVAL",
+ ]
+ .join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("1m"), || {
+ let opts = TestTaggedSeqConfig::load_layered_options(None, env_prefix).unwrap();
+ let defaults = TestTaggedSeqConfig::default();
+
+ // The entry is not in the default sequence, so it is appended.
+ assert_eq!(opts.region_engine.len(), defaults.region_engine.len() + 1);
+ assert_eq!(opts.region_engine[0], defaults.region_engine[0]);
+ assert_eq!(opts.region_engine[1], defaults.region_engine[1]);
+ assert_eq!(
+ test_metric(&opts).flush_metadata_region_interval,
+ Duration::from_secs(60)
+ );
+ });
+ }
+
+ #[test]
+ fn test_tagged_seq_env_overrides_a_nested_field() {
+ let env_prefix = "TAGGED_SEQ_NESTED_UT";
+ let env_key = [env_prefix, "REGION_ENGINE", "MITO", "INDEX", "AUX_PATH"].join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("/tmp/aux"), || {
+ let opts = TestTaggedSeqConfig::load_layered_options(None, env_prefix).unwrap();
+
+ let mito = test_mito(&opts);
+ assert_eq!(mito.index.aux_path, "/tmp/aux");
+ assert_eq!(
+ mito.index.cache_percent,
+ TestIndexConfig::default().cache_percent
+ );
+ assert_eq!(
+ mito.global_write_buffer_size,
+ TestMitoConfig::default().global_write_buffer_size
+ );
+ });
+ }
+
+ #[test]
+ fn test_tagged_seq_env_rejects_whole_key() {
+ let env_prefix = "TAGGED_SEQ_NO_TAG_UT";
+ let env_key = [env_prefix, "REGION_ENGINE"].join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("mito"), || {
+ let text = load_error_text(env_prefix);
+ assert!(text.contains("region_engine"), "{text}");
+ assert!(text.contains("region_engine____"), "{text}");
+ });
+ }
+
+ #[test]
+ fn test_tagged_seq_env_rejects_numeric_index() {
+ let env_prefix = "TAGGED_SEQ_NUMERIC_UT";
+ let env_key = [
+ env_prefix,
+ "REGION_ENGINE",
+ "0",
+ "MITO",
+ "GLOBAL_WRITE_BUFFER_SIZE",
+ ]
+ .join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("4GB"), || {
+ let text = load_error_text(env_prefix);
+ assert!(
+ text.contains("region_engine.0.mito.global_write_buffer_size"),
+ "{text}"
+ );
+ assert!(text.contains("addressed by tag name"), "{text}");
+ });
+ }
+
+ #[test]
+ fn test_tagged_seq_env_rejects_scalar_and_field_under_same_key() {
+ let env_prefix = "TAGGED_SEQ_NOT_A_TABLE_UT";
+ let whole_key = [env_prefix, "REGION_ENGINE", "MITO", "INDEX"].join(ENV_VAR_SEP);
+ let below_key =
+ [env_prefix, "REGION_ENGINE", "MITO", "INDEX", "AUX_PATH"].join(ENV_VAR_SEP);
+
+ temp_env::with_vars(
+ [
+ (whole_key, Some("not-a-table")),
+ (below_key, Some("/tmp/aux")),
+ ],
+ || {
+ // One variable sets `index` as a whole, another sets a field below it. The
+ // conflict is reported as such instead of failing later inside serde with a
+ // type error that does not name the variables involved.
+ let text = load_error_text(env_prefix);
+ assert!(
+ text.contains("another environment variable sets 'region_engine.mito.index'"),
+ "{text}"
+ );
+ assert!(text.contains("region_engine.mito.index.aux_path"), "{text}");
+ },
+ );
+ }
+
+ #[test]
+ fn test_tagged_seq_env_rejects_signed_and_float_indices() {
+ for index in ["-1", "+1", "1e3"] {
+ let env_prefix = "TAGGED_SEQ_INDEX_SHAPE_UT";
+ let env_key = [
+ env_prefix,
+ "REGION_ENGINE",
+ index,
+ "MITO",
+ "GLOBAL_WRITE_BUFFER_SIZE",
+ ]
+ .join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("4GB"), || {
+ let text = load_error_text(env_prefix);
+ assert!(text.contains("addressed by tag name"), "{index}: {text}");
+ });
+ }
+ }
+
+ #[test]
+ fn test_tagged_seq_env_rejects_tag_without_field() {
+ let env_prefix = "TAGGED_SEQ_NO_FIELD_UT";
+ let env_key = [env_prefix, "REGION_ENGINE", "MITO"].join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("mito"), || {
+ let text = load_error_text(env_prefix);
+ assert!(text.contains("region_engine.mito"), "{text}");
+ assert!(text.contains("region_engine____"), "{text}");
+ });
+ }
+
+ #[test]
+ fn test_tagged_seq_env_rejects_field_below_scalar_default() {
+ // The `data_dir` field of the file engine defaults to a string, so nothing can be set
+ // below it.
+ let env_prefix = "TAGGED_SEQ_SCALAR_DEFAULT_UT";
+ let env_key = [env_prefix, "REGION_ENGINE", "FILE", "DATA_DIR", "SUB"].join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("/tmp/sub"), || {
+ let text = load_error_text(env_prefix);
+ assert!(text.contains("region_engine.file.data_dir"), "{text}");
+ assert!(text.contains("default value is not a table"), "{text}");
+ });
+ }
+
+ #[test]
+ fn test_tagged_seq_env_type_error_names_the_environment() {
+ let env_prefix = "TAGGED_SEQ_TYPE_ERROR_UT";
+ let env_key = [
+ env_prefix,
+ "REGION_ENGINE",
+ "MITO",
+ "INDEX",
+ "CACHE_PERCENT",
+ ]
+ .join(ENV_VAR_SEP);
+
+ temp_env::with_var(env_key, Some("not-a-number"), || {
+ // The override replaces the default value as the environment collected it, so the
+ // type error still says where that value comes from.
+ let text = load_error_text(env_prefix);
+ assert!(text.contains("in the environment"), "{text}");
+ });
+ }
+
+ #[test]
+ fn test_tagged_seq_key_must_be_top_level() {
+ let err = TestDottedKeyConfig::load_layered_options(None, "TAGGED_SEQ_DOTTED_KEY_UT")
+ .unwrap_err();
+ let text = err.to_string();
+ assert!(text.contains("region_engine.mito"), "{text}");
+ assert!(text.contains("top-level"), "{text}");
+ }
}
diff --git a/src/common/config/src/error.rs b/src/common/config/src/error.rs
index 9e788d1d300..d7f81014e05 100644
--- a/src/common/config/src/error.rs
+++ b/src/common/config/src/error.rs
@@ -65,6 +65,84 @@ pub enum Error {
#[snafu(implicit)]
location: Location,
},
+
+ #[snafu(display(
+ "Invalid environment variable key '{key}': '{target}' is a sequence that cannot be \
+ overridden as a whole; address one of its entries with '{target}____'"
+ ))]
+ TaggedSeqMissingTag {
+ key: String,
+ target: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display(
+ "Invalid environment variable key '{key}': '{tag}' is a numeric index, but '{target}' \
+ is addressed by tag name; use '{target}____' instead, e.g. \
+ '{target}__MITO__'"
+ ))]
+ TaggedSeqNumericTag {
+ key: String,
+ tag: String,
+ target: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display(
+ "Invalid environment variable key '{key}': the entry '{tag}' of '{target}' needs a \
+ field to override; use '{target}____' instead"
+ ))]
+ TaggedSeqMissingField {
+ key: String,
+ tag: String,
+ target: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display(
+ "Invalid environment variable key '{key}': another environment variable sets '{path}' \
+ as a whole, so this key cannot set a field below it; set either the whole '{path}' or \
+ its fields, not both"
+ ))]
+ TaggedSeqOverridesConflict {
+ key: String,
+ path: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display(
+ "'{path}' cannot be overridden from the environment: its default value is not a table, \
+ so a field below it cannot be set"
+ ))]
+ TaggedSeqNotATable {
+ path: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display(
+ "'{key}' is registered as a sequence of tagged tables, but it is not a top-level field \
+ of the options; the default entries to merge onto are looked up by literal key"
+ ))]
+ TaggedSeqNotTopLevel {
+ key: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
+
+ #[snafu(display(
+ "The default value of '{target}' is not a sequence of single-key tables: {reason}"
+ ))]
+ TaggedSeqInvalidDefault {
+ target: String,
+ reason: String,
+ #[snafu(implicit)]
+ location: Location,
+ },
}
impl ErrorExt for Error {
@@ -73,7 +151,14 @@ impl ErrorExt for Error {
Error::TomlFormat { .. }
| Error::LoadLayeredConfig { .. }
| Error::FileWatch { .. }
- | Error::InvalidPath { .. } => StatusCode::InvalidArguments,
+ | Error::InvalidPath { .. }
+ | Error::TaggedSeqMissingTag { .. }
+ | Error::TaggedSeqNumericTag { .. }
+ | Error::TaggedSeqMissingField { .. }
+ | Error::TaggedSeqOverridesConflict { .. }
+ | Error::TaggedSeqNotATable { .. }
+ | Error::TaggedSeqNotTopLevel { .. }
+ | Error::TaggedSeqInvalidDefault { .. } => StatusCode::InvalidArguments,
Error::SerdeJson { .. } => StatusCode::Unexpected,
}
}
diff --git a/src/common/config/src/lib.rs b/src/common/config/src/lib.rs
index eea3b1351d8..7dc962fe5b1 100644
--- a/src/common/config/src/lib.rs
+++ b/src/common/config/src/lib.rs
@@ -15,6 +15,7 @@
pub mod config;
pub mod error;
pub mod file_watcher;
+mod tagged_seq;
use std::time::Duration;
diff --git a/src/common/config/src/tagged_seq.rs b/src/common/config/src/tagged_seq.rs
new file mode 100644
index 00000000000..7d2faa0b9bd
--- /dev/null
+++ b/src/common/config/src/tagged_seq.rs
@@ -0,0 +1,711 @@
+// Copyright 2023 Greptime Team
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Environment overrides for configuration keys holding a sequence of tagged tables.
+//!
+//! Such a sequence is what a `Vec` of externally tagged enums serializes to, for
+//! example the `region_engine` option:
+//!
+//! ```text
+//! [{"mito": {...}}, {"file": {...}}]
+//! ```
+//!
+//! Handing the plain [`Environment`] source to `config` cannot work for these keys:
+//! `config` writes every flat dotted key by walking down its path, and the walk resets
+//! any intermediate value that is not a table. Walking `region_engine.mito.index.aux_path`
+//! therefore destroys the default sequence before anything is set.
+//!
+//! This source instead addresses the entries by their tag name and merges the
+//! overrides onto the matching default entries:
+//!
+//! ```text
+//! GREPTIMEDB_STANDALONE__REGION_ENGINE__MITO__INDEX__AUX_PATH=...
+//! ```
+//!
+//! Entries that only the environment mentions are appended, and every field that is
+//! not mentioned keeps its default value. A configuration file that configures the
+//! whole sequence replaces the merged one, which makes these variables ineffective;
+//! the file has to spell out every entry it wants.
+
+use config::{ConfigError, Environment, Map, Source, Value, ValueKind};
+use serde_json::Value as JsonValue;
+
+use crate::error::{
+ Error, Result, TaggedSeqInvalidDefaultSnafu, TaggedSeqMissingFieldSnafu,
+ TaggedSeqMissingTagSnafu, TaggedSeqNotATableSnafu, TaggedSeqNumericTagSnafu,
+ TaggedSeqOverridesConflictSnafu,
+};
+
+/// A [`Source`] that merges environment overrides into sequences of tagged tables.
+///
+/// See the [module documentation](self) for the addressed syntax.
+#[derive(Debug)]
+pub(crate) struct TaggedSeqEnvSource {
+ /// The wrapped environment source. Everything it collects is kept as is, apart
+ /// from the entries that address one of the `targets`.
+ inner: Environment,
+ /// The keys whose entries are addressed by tag name.
+ targets: Vec,
+}
+
+/// One sequence of tagged tables that the environment may override.
+#[derive(Debug, Clone)]
+struct Target {
+ /// The dotted key of the sequence, e.g. `region_engine`.
+ key: String,
+ /// The default value of `key`, whose entries the overrides are merged into.
+ default: Option,
+}
+
+impl TaggedSeqEnvSource {
+ /// Wraps `inner` so that the sequence at every key of `keys` accepts tag name
+ /// overrides from the environment, merged onto the value that the same key has
+ /// in `defaults`.
+ pub(crate) fn new(
+ inner: Environment,
+ keys: &'static [&'static str],
+ defaults: &JsonValue,
+ ) -> Self {
+ let targets = keys
+ .iter()
+ .map(|key| Target {
+ key: (*key).to_string(),
+ default: defaults.get(key).cloned(),
+ })
+ .collect();
+
+ Self { inner, targets }
+ }
+}
+
+impl Source for TaggedSeqEnvSource {
+ fn clone_into_box(&self) -> Box {
+ Box::new(Self {
+ inner: self.inner.clone(),
+ targets: self.targets.clone(),
+ })
+ }
+
+ fn collect(&self) -> std::result::Result