common_wal/config/
object_store.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::time::Duration;
16
17use common_base::readable_size::ReadableSize;
18use serde::{Deserialize, Serialize};
19
20/// Generation of the node prefix in standalone mode.
21///
22/// Standalone runs a single datanode that never changes generation, so the
23/// prefix layout carries the constant. Distributed mode will take both the
24/// node id and the generation from the metasrv.
25pub const STANDALONE_GENERATION: u64 = 0;
26
27/// When an append to the object store WAL returns.
28#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
29#[serde(rename_all = "snake_case")]
30pub enum AckMode {
31    /// An append returns after the object holding its entries is durable.
32    #[default]
33    Durable,
34    /// An append returns once its entries are admitted and their ids are
35    /// assigned; the object is created in the background.
36    Enqueued,
37}
38
39/// What a read does with a segment that still does not decode after a
40/// second fetch, because its checksum does not match or its content
41/// disagrees with its footer entry.
42#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)]
43#[serde(rename_all = "snake_case")]
44pub enum CorruptedSegmentAction {
45    /// The segment is skipped and recorded as a hole of its region; the other
46    /// regions of the object are unaffected.
47    #[default]
48    Skip,
49    /// The read fails.
50    Fail,
51}
52
53/// Object store wal configurations for datanode.
54#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
55#[serde(default)]
56pub struct ObjectStoreWalConfig {
57    /// Name of the storage provider that holds the WAL objects.
58    /// An empty name selects the default object store.
59    pub storage_provider: String,
60    /// Path prefix of the WAL objects inside the storage provider.
61    ///
62    /// The store runs under a node prefix derived from it, see
63    /// [`node_prefix`](Self::node_prefix).
64    pub prefix: String,
65    /// Interval of flushing buffered entries to the object store, at least
66    /// 10ms, defaults to 100ms.
67    #[serde(with = "humantime_serde")]
68    pub flush_interval: Duration,
69    /// The max size of a single batch object, defaults to 8MiB.
70    pub max_batch_bytes: ReadableSize,
71    /// When an append returns, defaults to `durable`.
72    pub ack_mode: AckMode,
73    /// Size of the unpersisted backlog at which appends stall until an
74    /// upload completes in `enqueued` mode, defaults to 64MiB.
75    pub max_unpersisted_bytes: ReadableSize,
76    /// Age of the oldest unpersisted entry at which appends stall until an
77    /// upload completes in `enqueued` mode, defaults to 8s.
78    #[serde(with = "humantime_serde")]
79    pub max_unpersisted_age: Duration,
80    /// What a read does with a segment that still does not decode after a
81    /// second fetch, because its checksum does not match or its content
82    /// disagrees with its footer entry, defaults to `skip`.
83    pub on_corrupted_segment: CorruptedSegmentAction,
84}
85
86impl Default for ObjectStoreWalConfig {
87    fn default() -> Self {
88        Self {
89            storage_provider: String::new(),
90            prefix: "wal".to_string(),
91            flush_interval: Duration::from_millis(100),
92            max_batch_bytes: ReadableSize::mb(8),
93            ack_mode: AckMode::Durable,
94            max_unpersisted_bytes: ReadableSize::mb(64),
95            max_unpersisted_age: Duration::from_secs(8),
96            on_corrupted_segment: CorruptedSegmentAction::Skip,
97        }
98    }
99}
100
101impl ObjectStoreWalConfig {
102    /// Returns the prefix the store of `node_id` runs under in `generation`:
103    /// `<prefix>/datanodes/<node_id>/epochs/<generation>`.
104    ///
105    /// Every place that names the store's prefix, whether to open the store or
106    /// to allocate the WAL options of a region, must derive it here so the
107    /// prefix a region persists matches the one its store runs under.
108    pub fn node_prefix(&self, node_id: u64, generation: u64) -> String {
109        format!("{}/datanodes/{node_id}/epochs/{generation}", self.prefix)
110    }
111}
112
113#[cfg(test)]
114mod tests {
115    use super::*;
116
117    #[test]
118    fn test_node_prefix() {
119        let config = ObjectStoreWalConfig::default();
120        assert_eq!(
121            config.node_prefix(0, STANDALONE_GENERATION),
122            "wal/datanodes/0/epochs/0"
123        );
124
125        let config = ObjectStoreWalConfig {
126            prefix: "cluster-a/wal".to_string(),
127            ..Default::default()
128        };
129        assert_eq!(
130            config.node_prefix(3, 7),
131            "cluster-a/wal/datanodes/3/epochs/7"
132        );
133    }
134}
common_wal/config/object_store.rs

common_wal/config/
object_store.rs
