common_wal/
config.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub mod kafka;
16pub mod object_store;
17pub mod raft_engine;
18
19use std::time::Duration;
20
21use serde::{Deserialize, Serialize};
22
23use crate::config::kafka::common::{
24    DEFAULT_AUTO_PRUNE_INTERVAL, DEFAULT_AUTO_PRUNE_LOGICAL_DELETE, DEFAULT_AUTO_PRUNE_PARALLELISM,
25    DEFAULT_CHECKPOINT_TRIGGER_SIZE, DEFAULT_FLUSH_TRIGGER_SIZE,
26    DEFAULT_PERIODIC_CHECKPOINT_PERSIST_INTERVAL, DEFAULT_REGION_FLUSH_TRIGGER_INTERVAL,
27};
28use crate::config::kafka::{DatanodeKafkaConfig, MetasrvKafkaConfig};
29use crate::config::object_store::ObjectStoreWalConfig;
30use crate::config::raft_engine::RaftEngineConfig;
31use crate::error::{Error, UnsupportedWalProviderSnafu};
32
33/// Wal configurations for metasrv.
34#[derive(Debug, Serialize, Deserialize, Clone, PartialEq, Default)]
35#[serde(tag = "provider", rename_all = "snake_case")]
36#[allow(clippy::large_enum_variant)]
37pub enum MetasrvWalConfig {
38    #[default]
39    RaftEngine,
40    Kafka(MetasrvKafkaConfig),
41}
42
43#[allow(clippy::large_enum_variant)]
44/// Wal configurations for datanode.
45#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]
46#[serde(tag = "provider", rename_all = "snake_case")]
47pub enum DatanodeWalConfig {
48    RaftEngine(RaftEngineConfig),
49    Kafka(DatanodeKafkaConfig),
50    Noop,
51    #[serde(rename = "experimental_object_store")]
52    ObjectStore(ObjectStoreWalConfig),
53}
54
55impl Default for DatanodeWalConfig {
56    fn default() -> Self {
57        Self::RaftEngine(RaftEngineConfig::default())
58    }
59}
60
61impl TryFrom<DatanodeWalConfig> for MetasrvWalConfig {
62    type Error = Error;
63
64    fn try_from(config: DatanodeWalConfig) -> Result<Self, Self::Error> {
65        match config {
66            DatanodeWalConfig::RaftEngine(_) => Ok(Self::RaftEngine),
67            DatanodeWalConfig::Kafka(config) => Ok(Self::Kafka(MetasrvKafkaConfig {
68                connection: config.connection,
69                kafka_topic: config.kafka_topic,
70                auto_create_topics: config.auto_create_topics,
71                // This field won't be used in standalone mode
72                auto_prune_interval: DEFAULT_AUTO_PRUNE_INTERVAL,
73                // This field won't be used in standalone mode
74                auto_prune_logical_delete: DEFAULT_AUTO_PRUNE_LOGICAL_DELETE,
75                // This field won't be used in standalone mode
76                auto_prune_parallelism: DEFAULT_AUTO_PRUNE_PARALLELISM,
77                // This field won't be used in standalone mode
78                flush_trigger_size: DEFAULT_FLUSH_TRIGGER_SIZE,
79                // This field won't be used in standalone mode
80                checkpoint_trigger_size: DEFAULT_CHECKPOINT_TRIGGER_SIZE,
81                // This field won't be used in standalone mode
82                region_flush_trigger_interval: DEFAULT_REGION_FLUSH_TRIGGER_INTERVAL,
83                // This field won't be used in standalone mode
84                periodic_checkpoint_persist_interval: DEFAULT_PERIODIC_CHECKPOINT_PERSIST_INTERVAL,
85            })),
86            DatanodeWalConfig::Noop => UnsupportedWalProviderSnafu {
87                provider: "noop".to_string(),
88            }
89            .fail(),
90            DatanodeWalConfig::ObjectStore(_) => UnsupportedWalProviderSnafu {
91                provider: "experimental_object_store".to_string(),
92            }
93            .fail(),
94        }
95    }
96}
97
98impl MetasrvWalConfig {
99    /// Returns if active wal pruning is enabled.
100    pub fn enable_active_wal_pruning(&self) -> bool {
101        match self {
102            MetasrvWalConfig::RaftEngine => false,
103            MetasrvWalConfig::Kafka(config) => config.auto_prune_interval > Duration::ZERO,
104        }
105    }
106
107    /// Gets the kafka connection config.
108    pub fn remote_wal_options(&self) -> Option<&MetasrvKafkaConfig> {
109        match self {
110            MetasrvWalConfig::RaftEngine => None,
111            MetasrvWalConfig::Kafka(config) => Some(config),
112        }
113    }
114}
115
116impl From<MetasrvWalConfig> for DatanodeWalConfig {
117    fn from(config: MetasrvWalConfig) -> Self {
118        match config {
119            MetasrvWalConfig::RaftEngine => Self::RaftEngine(RaftEngineConfig::default()),
120            MetasrvWalConfig::Kafka(config) => Self::Kafka(DatanodeKafkaConfig {
121                connection: config.connection,
122                kafka_topic: config.kafka_topic,
123                ..Default::default()
124            }),
125        }
126    }
127}
128
129#[cfg(test)]
130mod tests {
131    use std::time::Duration;
132
133    use common_base::readable_size::ReadableSize;
134    use kafka::common::{
135        KafkaClientSasl, KafkaClientSaslConfig, KafkaClientTls, KafkaConnectionConfig,
136    };
137    use tests::kafka::common::KafkaTopicConfig;
138
139    use super::*;
140    use crate::TopicSelectorType;
141    use crate::config::object_store::{AckMode, CorruptedSegmentAction};
142    use crate::config::{DatanodeKafkaConfig, MetasrvKafkaConfig};
143
144    #[test]
145    fn test_toml_raft_engine() {
146        // With none configs.
147        let toml_str = r#"
148            provider = "raft_engine"
149        "#;
150        let metasrv_wal_config: MetasrvWalConfig = toml::from_str(toml_str).unwrap();
151        assert_eq!(metasrv_wal_config, MetasrvWalConfig::RaftEngine);
152
153        let datanode_wal_config: DatanodeWalConfig = toml::from_str(toml_str).unwrap();
154        assert_eq!(
155            datanode_wal_config,
156            DatanodeWalConfig::RaftEngine(RaftEngineConfig::default())
157        );
158
159        // With useless configs.
160        let toml_str = r#"
161            provider = "raft_engine"
162            broker_endpoints = ["127.0.0.1:9092"]
163            num_topics = 32
164        "#;
165        let datanode_wal_config: DatanodeWalConfig = toml::from_str(toml_str).unwrap();
166        assert_eq!(
167            datanode_wal_config,
168            DatanodeWalConfig::RaftEngine(RaftEngineConfig::default())
169        );
170
171        // With some useful configs.
172        let toml_str = r#"
173            provider = "raft_engine"
174            file_size = "4MB"
175            purge_threshold = "1GB"
176            purge_interval = "5mins"
177        "#;
178        let datanode_wal_config: DatanodeWalConfig = toml::from_str(toml_str).unwrap();
179        let expected = RaftEngineConfig {
180            file_size: ReadableSize::mb(4),
181            purge_threshold: ReadableSize::gb(1),
182            purge_interval: Duration::from_secs(5 * 60),
183            ..Default::default()
184        };
185        assert_eq!(datanode_wal_config, DatanodeWalConfig::RaftEngine(expected));
186    }
187
188    #[test]
189    fn test_toml_kafka() {
190        let toml_str = r#"
191            provider = "kafka"
192            broker_endpoints = ["127.0.0.1:9092"]
193            max_batch_bytes = "1MB"
194            consumer_wait_timeout = "100ms"
195            num_topics = 32
196            num_partitions = 1
197            selector_type = "round_robin"
198            replication_factor = 1
199            create_topic_timeout = "30s"
200            topic_name_prefix = "greptimedb_wal_topic"
201            [tls]
202            server_ca_cert_path = "/path/to/server.pem"
203            [sasl]
204            type = "SCRAM-SHA-512"
205            username = "hi"
206            password = "test"
207        "#;
208
209        // Deserialized to MetasrvWalConfig.
210        let metasrv_wal_config: MetasrvWalConfig = toml::from_str(toml_str).unwrap();
211        let expected = MetasrvKafkaConfig {
212            connection: KafkaConnectionConfig {
213                broker_endpoints: vec!["127.0.0.1:9092".to_string()],
214                sasl: Some(KafkaClientSasl {
215                    config: KafkaClientSaslConfig::ScramSha512 {
216                        username: "hi".to_string(),
217                        password: "test".to_string(),
218                    },
219                }),
220                tls: Some(KafkaClientTls {
221                    server_ca_cert_path: Some("/path/to/server.pem".to_string()),
222                    client_cert_path: None,
223                    client_key_path: None,
224                }),
225                connect_timeout: Duration::from_secs(3),
226                timeout: Duration::from_secs(5),
227            },
228            kafka_topic: KafkaTopicConfig {
229                num_topics: 32,
230                selector_type: TopicSelectorType::RoundRobin,
231                topic_name_prefix: "greptimedb_wal_topic".to_string(),
232                num_partitions: 1,
233                replication_factor: 1,
234                create_topic_timeout: Duration::from_secs(30),
235            },
236            auto_create_topics: true,
237            auto_prune_interval: Duration::from_secs(30 * 60),
238            auto_prune_logical_delete: false,
239            auto_prune_parallelism: 10,
240            flush_trigger_size: ReadableSize::mb(512),
241            checkpoint_trigger_size: ReadableSize::mb(128),
242            region_flush_trigger_interval: Duration::from_secs(60),
243            periodic_checkpoint_persist_interval: Duration::from_secs(60 * 60),
244        };
245        assert_eq!(metasrv_wal_config, MetasrvWalConfig::Kafka(expected));
246
247        // Deserialized to DatanodeWalConfig.
248        let datanode_wal_config: DatanodeWalConfig = toml::from_str(toml_str).unwrap();
249        let expected = DatanodeKafkaConfig {
250            connection: KafkaConnectionConfig {
251                broker_endpoints: vec!["127.0.0.1:9092".to_string()],
252                sasl: Some(KafkaClientSasl {
253                    config: KafkaClientSaslConfig::ScramSha512 {
254                        username: "hi".to_string(),
255                        password: "test".to_string(),
256                    },
257                }),
258                tls: Some(KafkaClientTls {
259                    server_ca_cert_path: Some("/path/to/server.pem".to_string()),
260                    client_cert_path: None,
261                    client_key_path: None,
262                }),
263                connect_timeout: Duration::from_secs(3),
264                timeout: Duration::from_secs(5),
265            },
266            max_batch_bytes: ReadableSize::mb(1),
267            consumer_wait_timeout: Duration::from_millis(100),
268            kafka_topic: KafkaTopicConfig {
269                num_topics: 32,
270                selector_type: TopicSelectorType::RoundRobin,
271                topic_name_prefix: "greptimedb_wal_topic".to_string(),
272                num_partitions: 1,
273                replication_factor: 1,
274                create_topic_timeout: Duration::from_secs(30),
275            },
276            ..Default::default()
277        };
278        assert_eq!(datanode_wal_config, DatanodeWalConfig::Kafka(expected));
279    }
280
281    #[test]
282    fn test_toml_object_store() {
283        let toml_str = r#"
284            provider = "experimental_object_store"
285        "#;
286        let datanode_wal_config: DatanodeWalConfig = toml::from_str(toml_str).unwrap();
287        assert_eq!(
288            datanode_wal_config,
289            DatanodeWalConfig::ObjectStore(ObjectStoreWalConfig::default())
290        );
291        let DatanodeWalConfig::ObjectStore(config) = &datanode_wal_config else {
292            unreachable!()
293        };
294        assert_eq!(config.prefix, "wal");
295        assert_eq!(config.flush_interval, Duration::from_millis(100));
296        assert_eq!(config.max_batch_bytes, ReadableSize::mb(8));
297        assert_eq!(config.ack_mode, AckMode::Durable);
298        assert_eq!(config.max_unpersisted_bytes, ReadableSize::mb(64));
299        assert_eq!(config.max_unpersisted_age, Duration::from_secs(8));
300        assert_eq!(config.on_corrupted_segment, CorruptedSegmentAction::Skip);
301        assert!(matches!(
302            MetasrvWalConfig::try_from(datanode_wal_config).unwrap_err(),
303            Error::UnsupportedWalProvider { .. }
304        ));
305
306        let toml_str = r#"
307            provider = "experimental_object_store"
308            storage_provider = "s3"
309            prefix = "cluster-a/wal"
310            flush_interval = "500ms"
311            max_batch_bytes = "4MB"
312            ack_mode = "enqueued"
313            max_unpersisted_bytes = "32MB"
314            max_unpersisted_age = "4s"
315            on_corrupted_segment = "fail"
316        "#;
317        let datanode_wal_config: DatanodeWalConfig = toml::from_str(toml_str).unwrap();
318        let expected = ObjectStoreWalConfig {
319            storage_provider: "s3".to_string(),
320            prefix: "cluster-a/wal".to_string(),
321            flush_interval: Duration::from_millis(500),
322            max_batch_bytes: ReadableSize::mb(4),
323            ack_mode: AckMode::Enqueued,
324            max_unpersisted_bytes: ReadableSize::mb(32),
325            max_unpersisted_age: Duration::from_secs(4),
326            on_corrupted_segment: CorruptedSegmentAction::Fail,
327        };
328        assert_eq!(
329            datanode_wal_config,
330            DatanodeWalConfig::ObjectStore(expected)
331        );
332        let serialized = toml::to_string(&datanode_wal_config).unwrap();
333        let table: toml::Table = toml::from_str(&serialized).unwrap();
334        assert_eq!(table["ack_mode"].as_str(), Some("enqueued"));
335        assert_eq!(table["max_unpersisted_bytes"].as_str(), Some("32MiB"));
336        assert_eq!(table["max_unpersisted_age"].as_str(), Some("4s"));
337        assert_eq!(
338            datanode_wal_config,
339            toml::from_str::<DatanodeWalConfig>(&serialized).unwrap()
340        );
341
342        // The persisted tag is not accepted as a config provider.
343        let toml_str = r#"
344            provider = "object_store"
345        "#;
346        assert!(toml::from_str::<DatanodeWalConfig>(toml_str).is_err());
347    }
348
349    #[test]
350    fn test_kafka_wal_config_debug_redacts_password() {
351        let config = MetasrvWalConfig::Kafka(MetasrvKafkaConfig {
352            connection: KafkaConnectionConfig {
353                sasl: Some(KafkaClientSasl {
354                    config: KafkaClientSaslConfig::Plain {
355                        username: "greptime".to_string(),
356                        password: "kafka-secret".to_string(),
357                    },
358                }),
359                ..Default::default()
360            },
361            ..Default::default()
362        });
363
364        let debug = format!("{config:#?}");
365
366        assert!(debug.contains("greptime"));
367        assert!(debug.contains("<REDACTED>"));
368        assert!(!debug.contains("kafka-secret"));
369    }
370}
371
372#[cfg(test)]
373mod rskafka_contract_tests {
374    use std::sync::Arc;
375    use std::time::Duration;
376
377    use rskafka::build_info::DEFAULT_CLIENT_ID;
378    use rskafka::messenger::{Messenger, RequestError, SyncVersionsError};
379
380    #[tokio::test]
381    async fn test_send_timeout_poisoned_messenger() {
382        let (_peer, stream) = tokio::io::duplex(1);
383        let mut messenger = Messenger::new(
384            stream,
385            1_000,
386            Arc::from(DEFAULT_CLIENT_ID),
387            Some(Duration::from_millis(50)),
388        );
389
390        let error = messenger.sync_versions().await.unwrap_err();
391        assert!(matches!(
392            error,
393            SyncVersionsError::RequestError(RequestError::Poisoned(error))
394                if matches!(
395                    error.as_ref(),
396                    RequestError::IO(error) if error.kind() == std::io::ErrorKind::TimedOut
397                )
398        ));
399    }
400}
common_wal/config.rs

common_wal/
config.rs
