store_api/
mito_engine_options.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Option keys for the mito engine.
16//! We define them in this mod so the create parser can use it to validate table options.
17
18use std::collections::HashMap;
19
20/// Option key for all WAL options.
21pub use common_wal::options::WAL_OPTIONS_KEY;
22/// Option key for append mode.
23pub const APPEND_MODE_KEY: &str = "append_mode";
24/// Option key for the per-region write buffer size.
25pub const WRITE_BUFFER_SIZE_KEY: &str = "write_buffer_size";
26/// Option key for merge mode.
27pub const MERGE_MODE_KEY: &str = "merge_mode";
28/// Option key for TTL(time-to-live)
29pub const TTL_KEY: &str = "ttl";
30/// Option key for the per-table auto flush interval.
31pub const AUTO_FLUSH_INTERVAL_KEY: &str = "auto_flush_interval";
32/// Option key for snapshot read.
33pub const SNAPSHOT_READ: &str = "snapshot_read";
34/// Option key for compaction type.
35pub const COMPACTION_TYPE: &str = "compaction.type";
36/// Option key for forcing compaction options override.
37pub const COMPACTION_OVERRIDE: &str = "compaction.override";
38/// TWCS compaction strategy.
39pub const COMPACTION_TYPE_TWCS: &str = "twcs";
40/// Option key for twcs min file num to trigger a compaction.
41pub const TWCS_TRIGGER_FILE_NUM: &str = "compaction.twcs.trigger_file_num";
42/// Option key for twcs min file num to trigger compaction in the active window.
43pub const TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM: &str =
44    "compaction.twcs.active_window.trigger_file_num";
45/// Option key for the active-window L1 safety compaction threshold.
46pub const TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER: &str =
47    "compaction.twcs.active_window.l1_merge_trigger";
48/// Option key for twcs min file num to trigger compaction in an inactive window.
49pub const TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM: &str =
50    "compaction.twcs.inactive_window.trigger_file_num";
51/// Option key for the inactive-window L1 compaction threshold.
52pub const TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER: &str =
53    "compaction.twcs.inactive_window.l1_merge_trigger";
54/// Option key for twcs max output file size.
55pub const TWCS_MAX_OUTPUT_FILE_SIZE: &str = "compaction.twcs.max_output_file_size";
56/// Option key for twcs time window.
57pub const TWCS_TIME_WINDOW: &str = "compaction.twcs.time_window";
58/// Option key for twcs remote compaction.
59pub const TWCS_REMOTE_COMPACTION: &str = "compaction.twcs.remote_compaction";
60/// Option key for twcs fallback to local.
61pub const TWCS_FALLBACK_TO_LOCAL: &str = "compaction.twcs.fallback_to_local";
62/// Option key for memtable type.
63pub const MEMTABLE_TYPE: &str = "memtable.type";
64/// Option key for bulk memtable merge threshold.
65pub const MEMTABLE_BULK_MERGE_THRESHOLD: &str = "memtable.bulk.merge_threshold";
66/// Option key for bulk memtable encode row threshold.
67pub const MEMTABLE_BULK_ENCODE_ROW_THRESHOLD: &str = "memtable.bulk.encode_row_threshold";
68/// Option key for bulk memtable encode bytes threshold.
69pub const MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD: &str = "memtable.bulk.encode_bytes_threshold";
70/// Option key for bulk memtable max merge groups.
71pub const MEMTABLE_BULK_MAX_MERGE_GROUPS: &str = "memtable.bulk.max_merge_groups";
72/// Option key for memtable partition tree index max keys per shard.
73pub const MEMTABLE_PARTITION_TREE_INDEX_MAX_KEYS_PER_SHARD: &str =
74    "memtable.partition_tree.index_max_keys_per_shard";
75/// Option key for memtable partition tree data freeze threshold.
76pub const MEMTABLE_PARTITION_TREE_DATA_FREEZE_THRESHOLD: &str =
77    "memtable.partition_tree.data_freeze_threshold";
78/// Option key for memtable partition tree fork dictionary bytes.
79pub const MEMTABLE_PARTITION_TREE_FORK_DICTIONARY_BYTES: &str =
80    "memtable.partition_tree.fork_dictionary_bytes";
81/// Option key for skipping WAL.
82pub const SKIP_WAL_KEY: &str = "skip_wal";
83/// Option key for sst format.
84pub const SST_FORMAT_KEY: &str = "sst_format";
85/// Option key for the max number of rows in a parquet row group.
86pub const MAX_ROW_GROUP_ROW_COUNT: &str = "max_row_group_row_count";
87/// Upper bound for [`MAX_ROW_GROUP_ROW_COUNT`].
88pub const MAX_ROW_GROUP_ROW_COUNT_LIMIT: usize = 10 * 1024 * 1024;
89/// Option key for preserving per-row sequence numbers through flush and compaction.
90pub const PRESERVE_ROW_SEQUENCE: &str = "preserve_row_sequence";
91/// Option key for experimental Parquet float field encoding.
92pub const EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING: &str = "experimental_sst_float_field_encoding";
93
94/// Encoding policy for direct floating-point field columns in Parquet SSTs.
95#[derive(
96    Debug,
97    Default,
98    Clone,
99    Copy,
100    PartialEq,
101    Eq,
102    Hash,
103    serde::Serialize,
104    serde::Deserialize,
105    strum::EnumString,
106)]
107#[serde(rename_all = "snake_case")]
108#[strum(serialize_all = "snake_case")]
109pub enum FloatFieldEncoding {
110    /// The existing Parquet writer behavior.
111    #[default]
112    Default,
113    /// Parquet byte-stream-split encoding.
114    ByteStreamSplit,
115}
116// Note: Adding new options here should also check if this option should be removed in [metric_engine::engine::create::region_options_for_metadata_region].
117
118/// Conflicting values supplied through the legacy and canonical TWCS trigger options.
119#[derive(Debug, PartialEq, Eq)]
120pub struct TwcsTriggerOptionConflict {
121    /// Value supplied under [`TWCS_TRIGGER_FILE_NUM`].
122    pub legacy_value: String,
123    /// Value supplied under [`TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM`].
124    pub canonical_value: String,
125}
126
127/// Normalizes the active-window TWCS trigger option to its legacy storage key.
128pub fn normalize_twcs_trigger_options(
129    options: &mut HashMap<String, String>,
130) -> Result<(), TwcsTriggerOptionConflict> {
131    let Some(canonical_value) = options.get(TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM).cloned() else {
132        return Ok(());
133    };
134    if let Some(legacy_value) = options.get(TWCS_TRIGGER_FILE_NUM)
135        && legacy_value != &canonical_value
136    {
137        return Err(TwcsTriggerOptionConflict {
138            legacy_value: legacy_value.clone(),
139            canonical_value,
140        });
141    }
142
143    options.remove(TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM);
144    options.insert(TWCS_TRIGGER_FILE_NUM.to_string(), canonical_value);
145    Ok(())
146}
147
148/// Returns true if the `key` is a valid option key for the mito engine.
149pub fn is_mito_engine_option_key(key: &str) -> bool {
150    [
151        "ttl",
152        WRITE_BUFFER_SIZE_KEY,
153        AUTO_FLUSH_INTERVAL_KEY,
154        COMPACTION_TYPE,
155        COMPACTION_OVERRIDE,
156        TWCS_TRIGGER_FILE_NUM,
157        TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM,
158        TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER,
159        TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM,
160        TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER,
161        TWCS_MAX_OUTPUT_FILE_SIZE,
162        TWCS_TIME_WINDOW,
163        TWCS_REMOTE_COMPACTION,
164        TWCS_FALLBACK_TO_LOCAL,
165        "storage",
166        "index.inverted_index.ignore_column_ids",
167        "index.inverted_index.segment_row_count",
168        WAL_OPTIONS_KEY,
169        MEMTABLE_TYPE,
170        MEMTABLE_BULK_MERGE_THRESHOLD,
171        MEMTABLE_BULK_ENCODE_ROW_THRESHOLD,
172        MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD,
173        MEMTABLE_BULK_MAX_MERGE_GROUPS,
174        MEMTABLE_PARTITION_TREE_INDEX_MAX_KEYS_PER_SHARD,
175        MEMTABLE_PARTITION_TREE_DATA_FREEZE_THRESHOLD,
176        MEMTABLE_PARTITION_TREE_FORK_DICTIONARY_BYTES,
177        // We don't allow to create a mito table with sparse primary key encoding directly.
178        APPEND_MODE_KEY,
179        MERGE_MODE_KEY,
180        SST_FORMAT_KEY,
181        MAX_ROW_GROUP_ROW_COUNT,
182        PRESERVE_ROW_SEQUENCE,
183        EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING,
184    ]
185    .contains(&key)
186}
187
188#[cfg(test)]
189mod tests {
190    use std::collections::HashMap;
191
192    use super::*;
193
194    #[test]
195    fn test_is_mito_engine_option_key() {
196        assert!(is_mito_engine_option_key("ttl"));
197        assert!(is_mito_engine_option_key("write_buffer_size"));
198        assert!(is_mito_engine_option_key("auto_flush_interval"));
199        assert!(is_mito_engine_option_key("compaction.type"));
200        assert!(is_mito_engine_option_key("compaction.override"));
201        assert!(is_mito_engine_option_key(
202            "compaction.twcs.trigger_file_num"
203        ));
204        assert!(is_mito_engine_option_key(
205            "compaction.twcs.active_window.trigger_file_num"
206        ));
207        assert!(is_mito_engine_option_key(
208            "compaction.twcs.active_window.l1_merge_trigger"
209        ));
210        assert!(is_mito_engine_option_key(
211            "compaction.twcs.inactive_window.trigger_file_num"
212        ));
213        assert!(is_mito_engine_option_key(
214            "compaction.twcs.inactive_window.l1_merge_trigger"
215        ));
216        assert!(is_mito_engine_option_key("compaction.twcs.time_window"));
217        assert!(is_mito_engine_option_key("storage"));
218        assert!(is_mito_engine_option_key(
219            "index.inverted_index.ignore_column_ids"
220        ));
221        assert!(is_mito_engine_option_key(
222            "index.inverted_index.segment_row_count"
223        ));
224        assert!(is_mito_engine_option_key("wal_options"));
225        assert!(is_mito_engine_option_key("memtable.type"));
226        assert!(is_mito_engine_option_key("memtable.bulk.merge_threshold"));
227        assert!(is_mito_engine_option_key(
228            "memtable.bulk.encode_row_threshold"
229        ));
230        assert!(is_mito_engine_option_key(
231            "memtable.bulk.encode_bytes_threshold"
232        ));
233        assert!(is_mito_engine_option_key("memtable.bulk.max_merge_groups"));
234        assert!(is_mito_engine_option_key(
235            "memtable.partition_tree.index_max_keys_per_shard"
236        ));
237        assert!(is_mito_engine_option_key(
238            "memtable.partition_tree.data_freeze_threshold"
239        ));
240        assert!(is_mito_engine_option_key(
241            "memtable.partition_tree.fork_dictionary_bytes"
242        ));
243        assert!(is_mito_engine_option_key("append_mode"));
244        assert!(is_mito_engine_option_key("max_row_group_row_count"));
245        assert!(is_mito_engine_option_key("preserve_row_sequence"));
246        assert!(is_mito_engine_option_key(
247            EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING
248        ));
249        assert!(!is_mito_engine_option_key("foo"));
250    }
251
252    #[test]
253    fn test_float_field_encoding_values() {
254        assert_eq!(
255            "default".parse::<FloatFieldEncoding>(),
256            Ok(FloatFieldEncoding::Default)
257        );
258        assert_eq!(
259            "byte_stream_split".parse::<FloatFieldEncoding>(),
260            Ok(FloatFieldEncoding::ByteStreamSplit)
261        );
262        assert_eq!(
263            serde_json::from_str::<FloatFieldEncoding>("\"byte_stream_split\"").unwrap(),
264            FloatFieldEncoding::ByteStreamSplit
265        );
266        assert!(serde_json::from_str::<FloatFieldEncoding>("\"unknown\"").is_err());
267    }
268
269    #[test]
270    fn test_normalize_twcs_trigger_aliases_to_legacy_key() {
271        let expected = HashMap::from([(TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string())]);
272        for mut options in [
273            HashMap::from([(TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string())]),
274            HashMap::from([(
275                TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM.to_string(),
276                "4".to_string(),
277            )]),
278            HashMap::from([
279                (TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string()),
280                (
281                    TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM.to_string(),
282                    "4".to_string(),
283                ),
284            ]),
285        ] {
286            normalize_twcs_trigger_options(&mut options).unwrap();
287            assert_eq!(expected, options);
288        }
289    }
290
291    #[test]
292    fn test_normalize_twcs_trigger_conflicting_aliases() {
293        let mut options = HashMap::from([
294            (TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string()),
295            (
296                TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM.to_string(),
297                "8".to_string(),
298            ),
299        ]);
300        let original = options.clone();
301
302        let error = normalize_twcs_trigger_options(&mut options).unwrap_err();
303
304        assert_eq!("4", error.legacy_value);
305        assert_eq!("8", error.canonical_value);
306        assert_eq!(original, options);
307    }
308}
store_api/mito_engine_options.rs

store_api/
mito_engine_options.rs
