common_wal/
config.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub mod kafka;
16pub mod raft_engine;
17
18use std::time::Duration;
19
20use serde::{Deserialize, Serialize};
21
22use crate::config::kafka::common::{
23    DEFAULT_AUTO_PRUNE_INTERVAL, DEFAULT_AUTO_PRUNE_LOGICAL_DELETE, DEFAULT_AUTO_PRUNE_PARALLELISM,
24    DEFAULT_CHECKPOINT_TRIGGER_SIZE, DEFAULT_FLUSH_TRIGGER_SIZE,
25    DEFAULT_PERIODIC_CHECKPOINT_PERSIST_INTERVAL, DEFAULT_REGION_FLUSH_TRIGGER_INTERVAL,
26};
27use crate::config::kafka::{DatanodeKafkaConfig, MetasrvKafkaConfig};
28use crate::config::raft_engine::RaftEngineConfig;
29use crate::error::{Error, UnsupportedWalProviderSnafu};
30
31/// Wal configurations for metasrv.
32#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
33#[serde(tag = "provider", rename_all = "snake_case")]
34#[allow(clippy::large_enum_variant)]
35pub enum MetasrvWalConfig {
36    #[default]
37    RaftEngine,
38    Kafka(MetasrvKafkaConfig),
39}
40
41#[allow(clippy::large_enum_variant)]
42/// Wal configurations for datanode.
43#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
44#[serde(tag = "provider", rename_all = "snake_case")]
45pub enum DatanodeWalConfig {
46    RaftEngine(RaftEngineConfig),
47    Kafka(DatanodeKafkaConfig),
48    Noop,
49}
50
51impl Default for DatanodeWalConfig {
52    fn default() -> Self {
53        Self::RaftEngine(RaftEngineConfig::default())
54    }
55}
56
57impl TryFrom<DatanodeWalConfig> for MetasrvWalConfig {
58    type Error = Error;
59
60    fn try_from(config: DatanodeWalConfig) -> Result<Self, Self::Error> {
61        match config {
62            DatanodeWalConfig::RaftEngine(_) => Ok(Self::RaftEngine),
63            DatanodeWalConfig::Kafka(config) => Ok(Self::Kafka(MetasrvKafkaConfig {
64                connection: config.connection,
65                kafka_topic: config.kafka_topic,
66                auto_create_topics: config.auto_create_topics,
67                // This field won't be used in standalone mode
68                auto_prune_interval: DEFAULT_AUTO_PRUNE_INTERVAL,
69                // This field won't be used in standalone mode
70                auto_prune_logical_delete: DEFAULT_AUTO_PRUNE_LOGICAL_DELETE,
71                // This field won't be used in standalone mode
72                auto_prune_parallelism: DEFAULT_AUTO_PRUNE_PARALLELISM,
73                // This field won't be used in standalone mode
74                flush_trigger_size: DEFAULT_FLUSH_TRIGGER_SIZE,
75                // This field won't be used in standalone mode
76                checkpoint_trigger_size: DEFAULT_CHECKPOINT_TRIGGER_SIZE,
77                // This field won't be used in standalone mode
78                region_flush_trigger_interval: DEFAULT_REGION_FLUSH_TRIGGER_INTERVAL,
79                // This field won't be used in standalone mode
80                periodic_checkpoint_persist_interval: DEFAULT_PERIODIC_CHECKPOINT_PERSIST_INTERVAL,
81            })),
82            DatanodeWalConfig::Noop => UnsupportedWalProviderSnafu {
83                provider: "noop".to_string(),
84            }
85            .fail(),
86        }
87    }
88}
89
90impl MetasrvWalConfig {
91    /// Returns if active wal pruning is enabled.
92    pub fn enable_active_wal_pruning(&self) -> bool {
93        match self {
94            MetasrvWalConfig::RaftEngine => false,
95            MetasrvWalConfig::Kafka(config) => config.auto_prune_interval > Duration::ZERO,
96        }
97    }
98
99    /// Gets the kafka connection config.
100    pub fn remote_wal_options(&self) -> Option<&MetasrvKafkaConfig> {
101        match self {
102            MetasrvWalConfig::RaftEngine => None,
103            MetasrvWalConfig::Kafka(config) => Some(config),
104        }
105    }
106}
107
108impl From<MetasrvWalConfig> for DatanodeWalConfig {
109    fn from(config: MetasrvWalConfig) -> Self {
110        match config {
111            MetasrvWalConfig::RaftEngine => Self::RaftEngine(RaftEngineConfig::default()),
112            MetasrvWalConfig::Kafka(config) => Self::Kafka(DatanodeKafkaConfig {
113                connection: config.connection,
114                kafka_topic: config.kafka_topic,
115                ..Default::default()
116            }),
117        }
118    }
119}
120
121#[cfg(test)]
122mod tests {
123    use std::time::Duration;
124
125    use common_base::readable_size::ReadableSize;
126    use kafka::common::{
127        KafkaClientSasl, KafkaClientSaslConfig, KafkaClientTls, KafkaConnectionConfig,
128    };
129    use tests::kafka::common::KafkaTopicConfig;
130
131    use super::*;
132    use crate::TopicSelectorType;
133    use crate::config::{DatanodeKafkaConfig, MetasrvKafkaConfig};
134
135    #[test]
136    fn test_toml_raft_engine() {
137        // With none configs.
138        let toml_str = r#"
139            provider = "raft_engine"
140        "#;
141        let metasrv_wal_config: MetasrvWalConfig = toml::from_str(toml_str).unwrap();
142        assert_eq!(metasrv_wal_config, MetasrvWalConfig::RaftEngine);
143
144        let datanode_wal_config: DatanodeWalConfig = toml::from_str(toml_str).unwrap();
145        assert_eq!(
146            datanode_wal_config,
147            DatanodeWalConfig::RaftEngine(RaftEngineConfig::default())
148        );
149
150        // With useless configs.
151        let toml_str = r#"
152            provider = "raft_engine"
153            broker_endpoints = ["127.0.0.1:9092"]
154            num_topics = 32
155        "#;
156        let datanode_wal_config: DatanodeWalConfig = toml::from_str(toml_str).unwrap();
157        assert_eq!(
158            datanode_wal_config,
159            DatanodeWalConfig::RaftEngine(RaftEngineConfig::default())
160        );
161
162        // With some useful configs.
163        let toml_str = r#"
164            provider = "raft_engine"
165            file_size = "4MB"
166            purge_threshold = "1GB"
167            purge_interval = "5mins"
168        "#;
169        let datanode_wal_config: DatanodeWalConfig = toml::from_str(toml_str).unwrap();
170        let expected = RaftEngineConfig {
171            file_size: ReadableSize::mb(4),
172            purge_threshold: ReadableSize::gb(1),
173            purge_interval: Duration::from_secs(5 * 60),
174            ..Default::default()
175        };
176        assert_eq!(datanode_wal_config, DatanodeWalConfig::RaftEngine(expected));
177    }
178
179    #[test]
180    fn test_toml_kafka() {
181        let toml_str = r#"
182            provider = "kafka"
183            broker_endpoints = ["127.0.0.1:9092"]
184            max_batch_bytes = "1MB"
185            consumer_wait_timeout = "100ms"
186            num_topics = 32
187            num_partitions = 1
188            selector_type = "round_robin"
189            replication_factor = 1
190            create_topic_timeout = "30s"
191            topic_name_prefix = "greptimedb_wal_topic"
192            [tls]
193            server_ca_cert_path = "/path/to/server.pem"
194            [sasl]
195            type = "SCRAM-SHA-512"
196            username = "hi"
197            password = "test"
198        "#;
199
200        // Deserialized to MetasrvWalConfig.
201        let metasrv_wal_config: MetasrvWalConfig = toml::from_str(toml_str).unwrap();
202        let expected = MetasrvKafkaConfig {
203            connection: KafkaConnectionConfig {
204                broker_endpoints: vec!["127.0.0.1:9092".to_string()],
205                sasl: Some(KafkaClientSasl {
206                    config: KafkaClientSaslConfig::ScramSha512 {
207                        username: "hi".to_string(),
208                        password: "test".to_string(),
209                    },
210                }),
211                tls: Some(KafkaClientTls {
212                    server_ca_cert_path: Some("/path/to/server.pem".to_string()),
213                    client_cert_path: None,
214                    client_key_path: None,
215                }),
216                connect_timeout: Duration::from_secs(3),
217                timeout: Duration::from_secs(3),
218            },
219            kafka_topic: KafkaTopicConfig {
220                num_topics: 32,
221                selector_type: TopicSelectorType::RoundRobin,
222                topic_name_prefix: "greptimedb_wal_topic".to_string(),
223                num_partitions: 1,
224                replication_factor: 1,
225                create_topic_timeout: Duration::from_secs(30),
226            },
227            auto_create_topics: true,
228            auto_prune_interval: Duration::from_mins(30),
229            auto_prune_logical_delete: false,
230            auto_prune_parallelism: 10,
231            flush_trigger_size: ReadableSize::mb(512),
232            checkpoint_trigger_size: ReadableSize::mb(128),
233            region_flush_trigger_interval: Duration::from_secs(60),
234            periodic_checkpoint_persist_interval: Duration::from_secs(60 * 60),
235        };
236        assert_eq!(metasrv_wal_config, MetasrvWalConfig::Kafka(expected));
237
238        // Deserialized to DatanodeWalConfig.
239        let datanode_wal_config: DatanodeWalConfig = toml::from_str(toml_str).unwrap();
240        let expected = DatanodeKafkaConfig {
241            connection: KafkaConnectionConfig {
242                broker_endpoints: vec!["127.0.0.1:9092".to_string()],
243                sasl: Some(KafkaClientSasl {
244                    config: KafkaClientSaslConfig::ScramSha512 {
245                        username: "hi".to_string(),
246                        password: "test".to_string(),
247                    },
248                }),
249                tls: Some(KafkaClientTls {
250                    server_ca_cert_path: Some("/path/to/server.pem".to_string()),
251                    client_cert_path: None,
252                    client_key_path: None,
253                }),
254                connect_timeout: Duration::from_secs(3),
255                timeout: Duration::from_secs(3),
256            },
257            max_batch_bytes: ReadableSize::mb(1),
258            consumer_wait_timeout: Duration::from_millis(100),
259            kafka_topic: KafkaTopicConfig {
260                num_topics: 32,
261                selector_type: TopicSelectorType::RoundRobin,
262                topic_name_prefix: "greptimedb_wal_topic".to_string(),
263                num_partitions: 1,
264                replication_factor: 1,
265                create_topic_timeout: Duration::from_secs(30),
266            },
267            ..Default::default()
268        };
269        assert_eq!(datanode_wal_config, DatanodeWalConfig::Kafka(expected));
270    }
271
272    #[test]
273    fn test_kafka_wal_config_debug_redacts_password() {
274        let config = MetasrvWalConfig::Kafka(MetasrvKafkaConfig {
275            connection: KafkaConnectionConfig {
276                sasl: Some(KafkaClientSasl {
277                    config: KafkaClientSaslConfig::Plain {
278                        username: "greptime".to_string(),
279                        password: "kafka-secret".to_string(),
280                    },
281                }),
282                ..Default::default()
283            },
284            ..Default::default()
285        });
286
287        let debug = format!("{config:#?}");
288
289        assert!(debug.contains("greptime"));
290        assert!(debug.contains("<REDACTED>"));
291        assert!(!debug.contains("kafka-secret"));
292    }
293}
common_wal/config.rs

common_wal/
config.rs
