store_api/
mito_engine_options.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Option keys for the mito engine.
16//! We define them in this mod so the create parser can use it to validate table options.
17
18use std::collections::HashMap;
19
20/// Option key for all WAL options.
21pub use common_wal::options::WAL_OPTIONS_KEY;
22/// Option key for append mode.
23pub const APPEND_MODE_KEY: &str = "append_mode";
24/// Option key for the per-region write buffer size.
25pub const WRITE_BUFFER_SIZE_KEY: &str = "write_buffer_size";
26/// Option key for merge mode.
27pub const MERGE_MODE_KEY: &str = "merge_mode";
28/// Option key for TTL(time-to-live)
29pub const TTL_KEY: &str = "ttl";
30/// Option key for the per-table auto flush interval.
31pub const AUTO_FLUSH_INTERVAL_KEY: &str = "auto_flush_interval";
32/// Per-table override for each compaction output's decode and encode parallelism.
33pub const COMPACTION_EXPERIMENTAL_OUTPUT_PARALLELISM: &str =
34    "compaction.experimental_output_parallelism";
35/// Option key for compaction type.
36pub const COMPACTION_TYPE: &str = "compaction.type";
37/// Option key for forcing compaction options override.
38pub const COMPACTION_OVERRIDE: &str = "compaction.override";
39/// TWCS compaction strategy.
40pub const COMPACTION_TYPE_TWCS: &str = "twcs";
41/// Prefix for TWCS compaction options.
42pub const COMPACTION_TWCS_PREFIX: &str = "compaction.twcs.";
43/// Option key for twcs min file num to trigger a compaction.
44pub const TWCS_TRIGGER_FILE_NUM: &str = "compaction.twcs.trigger_file_num";
45/// Option key for twcs min file num to trigger compaction in the active window.
46pub const TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM: &str =
47    "compaction.twcs.active_window.trigger_file_num";
48/// Option key for the active-window L1 safety compaction threshold.
49pub const TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER: &str =
50    "compaction.twcs.active_window.l1_merge_trigger";
51/// Option key for twcs min file num to trigger compaction in an inactive window.
52pub const TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM: &str =
53    "compaction.twcs.inactive_window.trigger_file_num";
54/// Option key for the inactive-window L1 compaction threshold.
55pub const TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER: &str =
56    "compaction.twcs.inactive_window.l1_merge_trigger";
57/// Option key for twcs max output file size.
58pub const TWCS_MAX_OUTPUT_FILE_SIZE: &str = "compaction.twcs.max_output_file_size";
59/// Option key for twcs time window.
60pub const TWCS_TIME_WINDOW: &str = "compaction.twcs.time_window";
61/// Option key for twcs remote compaction.
62pub const TWCS_REMOTE_COMPACTION: &str = "compaction.twcs.remote_compaction";
63/// Option key for twcs fallback to local.
64pub const TWCS_FALLBACK_TO_LOCAL: &str = "compaction.twcs.fallback_to_local";
65/// Table options that configure compaction.
66pub const COMPACTION_OPTION_KEYS: [&str; 11] = [
67    COMPACTION_TYPE,
68    COMPACTION_OVERRIDE,
69    TWCS_TRIGGER_FILE_NUM,
70    TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM,
71    TWCS_ACTIVE_WINDOW_L1_MERGE_TRIGGER,
72    TWCS_INACTIVE_WINDOW_TRIGGER_FILE_NUM,
73    TWCS_INACTIVE_WINDOW_L1_MERGE_TRIGGER,
74    TWCS_MAX_OUTPUT_FILE_SIZE,
75    TWCS_TIME_WINDOW,
76    TWCS_REMOTE_COMPACTION,
77    TWCS_FALLBACK_TO_LOCAL,
78];
79/// Option key for memtable type.
80pub const MEMTABLE_TYPE: &str = "memtable.type";
81/// Option key for bulk memtable merge threshold.
82pub const MEMTABLE_BULK_MERGE_THRESHOLD: &str = "memtable.bulk.merge_threshold";
83/// Option key for bulk memtable encode row threshold.
84pub const MEMTABLE_BULK_ENCODE_ROW_THRESHOLD: &str = "memtable.bulk.encode_row_threshold";
85/// Option key for bulk memtable encode bytes threshold.
86pub const MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD: &str = "memtable.bulk.encode_bytes_threshold";
87/// Option key for bulk memtable max merge groups.
88pub const MEMTABLE_BULK_MAX_MERGE_GROUPS: &str = "memtable.bulk.max_merge_groups";
89/// Option key for memtable partition tree index max keys per shard.
90pub const MEMTABLE_PARTITION_TREE_INDEX_MAX_KEYS_PER_SHARD: &str =
91    "memtable.partition_tree.index_max_keys_per_shard";
92/// Option key for memtable partition tree data freeze threshold.
93pub const MEMTABLE_PARTITION_TREE_DATA_FREEZE_THRESHOLD: &str =
94    "memtable.partition_tree.data_freeze_threshold";
95/// Option key for memtable partition tree fork dictionary bytes.
96pub const MEMTABLE_PARTITION_TREE_FORK_DICTIONARY_BYTES: &str =
97    "memtable.partition_tree.fork_dictionary_bytes";
98/// Option key for skipping WAL.
99pub const SKIP_WAL_KEY: &str = "skip_wal";
100/// Option key for sst format.
101pub const SST_FORMAT_KEY: &str = "sst_format";
102/// Option key for the max number of rows in a parquet row group.
103pub const MAX_ROW_GROUP_ROW_COUNT: &str = "max_row_group_row_count";
104/// Upper bound for [`MAX_ROW_GROUP_ROW_COUNT`].
105pub const MAX_ROW_GROUP_ROW_COUNT_LIMIT: usize = 10 * 1024 * 1024;
106/// Option key for preserving per-row sequence numbers through flush and compaction.
107pub const PRESERVE_ROW_SEQUENCE: &str = "preserve_row_sequence";
108/// Option key for experimental Parquet float field encoding.
109pub const EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING: &str = "experimental_sst_float_field_encoding";
110
111/// Encoding policy for direct floating-point field columns in Parquet SSTs.
112#[derive(
113    Debug,
114    Default,
115    Clone,
116    Copy,
117    PartialEq,
118    Eq,
119    Hash,
120    serde::Serialize,
121    serde::Deserialize,
122    strum::EnumString,
123)]
124#[serde(rename_all = "snake_case")]
125#[strum(serialize_all = "snake_case")]
126pub enum FloatFieldEncoding {
127    /// The existing Parquet writer behavior.
128    #[default]
129    Default,
130    /// Parquet byte-stream-split encoding.
131    ByteStreamSplit,
132}
133// Note: Adding new options here should also check if this option should be removed in [metric_engine::engine::create::region_options_for_metadata_region].
134
135/// Conflicting values supplied through the legacy and canonical TWCS trigger options.
136#[derive(Debug, PartialEq, Eq)]
137pub struct TwcsTriggerOptionConflict {
138    /// Value supplied under [`TWCS_TRIGGER_FILE_NUM`].
139    pub legacy_value: String,
140    /// Value supplied under [`TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM`].
141    pub canonical_value: String,
142}
143
144/// Normalizes the active-window TWCS trigger option to its legacy storage key.
145pub fn normalize_twcs_trigger_options(
146    options: &mut HashMap<String, String>,
147) -> Result<(), TwcsTriggerOptionConflict> {
148    let Some(canonical_value) = options.get(TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM).cloned() else {
149        return Ok(());
150    };
151    if let Some(legacy_value) = options.get(TWCS_TRIGGER_FILE_NUM)
152        && legacy_value != &canonical_value
153    {
154        return Err(TwcsTriggerOptionConflict {
155            legacy_value: legacy_value.clone(),
156            canonical_value,
157        });
158    }
159
160    options.remove(TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM);
161    options.insert(TWCS_TRIGGER_FILE_NUM.to_string(), canonical_value);
162    Ok(())
163}
164
165/// Returns true if the `key` is a valid option key for the mito engine.
166pub fn is_mito_engine_option_key(key: &str) -> bool {
167    COMPACTION_OPTION_KEYS.contains(&key)
168        || [
169            "ttl",
170            WRITE_BUFFER_SIZE_KEY,
171            AUTO_FLUSH_INTERVAL_KEY,
172            COMPACTION_EXPERIMENTAL_OUTPUT_PARALLELISM,
173            "storage",
174            "index.inverted_index.ignore_column_ids",
175            "index.inverted_index.segment_row_count",
176            WAL_OPTIONS_KEY,
177            MEMTABLE_TYPE,
178            MEMTABLE_BULK_MERGE_THRESHOLD,
179            MEMTABLE_BULK_ENCODE_ROW_THRESHOLD,
180            MEMTABLE_BULK_ENCODE_BYTES_THRESHOLD,
181            MEMTABLE_BULK_MAX_MERGE_GROUPS,
182            MEMTABLE_PARTITION_TREE_INDEX_MAX_KEYS_PER_SHARD,
183            MEMTABLE_PARTITION_TREE_DATA_FREEZE_THRESHOLD,
184            MEMTABLE_PARTITION_TREE_FORK_DICTIONARY_BYTES,
185            // We don't allow to create a mito table with sparse primary key encoding directly.
186            APPEND_MODE_KEY,
187            MERGE_MODE_KEY,
188            SST_FORMAT_KEY,
189            MAX_ROW_GROUP_ROW_COUNT,
190            PRESERVE_ROW_SEQUENCE,
191            EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING,
192        ]
193        .contains(&key)
194}
195
196#[cfg(test)]
197mod tests {
198    use std::collections::HashMap;
199
200    use super::*;
201
202    #[test]
203    fn test_is_mito_engine_option_key() {
204        assert!(is_mito_engine_option_key("ttl"));
205        assert!(is_mito_engine_option_key("write_buffer_size"));
206        assert!(is_mito_engine_option_key("auto_flush_interval"));
207        assert!(is_mito_engine_option_key(
208            "compaction.experimental_output_parallelism"
209        ));
210        assert!(is_mito_engine_option_key("compaction.type"));
211        assert!(is_mito_engine_option_key("compaction.override"));
212        assert!(is_mito_engine_option_key(
213            "compaction.twcs.trigger_file_num"
214        ));
215        assert!(is_mito_engine_option_key(
216            "compaction.twcs.active_window.trigger_file_num"
217        ));
218        assert!(is_mito_engine_option_key(
219            "compaction.twcs.active_window.l1_merge_trigger"
220        ));
221        assert!(is_mito_engine_option_key(
222            "compaction.twcs.inactive_window.trigger_file_num"
223        ));
224        assert!(is_mito_engine_option_key(
225            "compaction.twcs.inactive_window.l1_merge_trigger"
226        ));
227        assert!(is_mito_engine_option_key("compaction.twcs.time_window"));
228        assert!(is_mito_engine_option_key("storage"));
229        assert!(is_mito_engine_option_key(
230            "index.inverted_index.ignore_column_ids"
231        ));
232        assert!(is_mito_engine_option_key(
233            "index.inverted_index.segment_row_count"
234        ));
235        assert!(is_mito_engine_option_key("wal_options"));
236        assert!(is_mito_engine_option_key("memtable.type"));
237        assert!(is_mito_engine_option_key("memtable.bulk.merge_threshold"));
238        assert!(is_mito_engine_option_key(
239            "memtable.bulk.encode_row_threshold"
240        ));
241        assert!(is_mito_engine_option_key(
242            "memtable.bulk.encode_bytes_threshold"
243        ));
244        assert!(is_mito_engine_option_key("memtable.bulk.max_merge_groups"));
245        assert!(is_mito_engine_option_key(
246            "memtable.partition_tree.index_max_keys_per_shard"
247        ));
248        assert!(is_mito_engine_option_key(
249            "memtable.partition_tree.data_freeze_threshold"
250        ));
251        assert!(is_mito_engine_option_key(
252            "memtable.partition_tree.fork_dictionary_bytes"
253        ));
254        assert!(is_mito_engine_option_key("append_mode"));
255        assert!(is_mito_engine_option_key("max_row_group_row_count"));
256        assert!(is_mito_engine_option_key("preserve_row_sequence"));
257        assert!(is_mito_engine_option_key(
258            EXPERIMENTAL_SST_FLOAT_FIELD_ENCODING
259        ));
260        assert!(!is_mito_engine_option_key("foo"));
261    }
262
263    #[test]
264    fn test_float_field_encoding_values() {
265        assert_eq!(
266            "default".parse::<FloatFieldEncoding>(),
267            Ok(FloatFieldEncoding::Default)
268        );
269        assert_eq!(
270            "byte_stream_split".parse::<FloatFieldEncoding>(),
271            Ok(FloatFieldEncoding::ByteStreamSplit)
272        );
273        assert_eq!(
274            serde_json::from_str::<FloatFieldEncoding>("\"byte_stream_split\"").unwrap(),
275            FloatFieldEncoding::ByteStreamSplit
276        );
277        assert!(serde_json::from_str::<FloatFieldEncoding>("\"unknown\"").is_err());
278    }
279
280    #[test]
281    fn test_normalize_twcs_trigger_aliases_to_legacy_key() {
282        let expected = HashMap::from([(TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string())]);
283        for mut options in [
284            HashMap::from([(TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string())]),
285            HashMap::from([(
286                TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM.to_string(),
287                "4".to_string(),
288            )]),
289            HashMap::from([
290                (TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string()),
291                (
292                    TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM.to_string(),
293                    "4".to_string(),
294                ),
295            ]),
296        ] {
297            normalize_twcs_trigger_options(&mut options).unwrap();
298            assert_eq!(expected, options);
299        }
300    }
301
302    #[test]
303    fn test_normalize_twcs_trigger_conflicting_aliases() {
304        let mut options = HashMap::from([
305            (TWCS_TRIGGER_FILE_NUM.to_string(), "4".to_string()),
306            (
307                TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM.to_string(),
308                "8".to_string(),
309            ),
310        ]);
311        let original = options.clone();
312
313        let error = normalize_twcs_trigger_options(&mut options).unwrap_err();
314
315        assert_eq!("4", error.legacy_value);
316        assert_eq!("8", error.canonical_value);
317        assert_eq!(original, options);
318    }
319}
store_api/mito_engine_options.rs

store_api/
mito_engine_options.rs
