mito2/
compaction.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15mod buckets;
16pub mod compactor;
17mod json2;
18mod last_non_null;
19pub mod memory_manager;
20mod overlap;
21pub mod picker;
22mod reader;
23pub mod run;
24mod scheduler;
25mod task;
26#[cfg(test)]
27mod test_util;
28mod twcs;
29mod unit;
30mod window;
31
32use std::any::Any;
33use std::collections::HashMap;
34
35use common_meta::key::SchemaMetadataManagerRef;
36use common_telemetry::{debug, error};
37use common_time::TimeToLive;
38use common_time::range::TimestampRange;
39pub(crate) use json2::{
40    Json2RewritePlans, collect_json2_rewrite_plans, rewrite_json2_batch, rewrite_json2_schema,
41};
42pub use scheduler::CompactionRequest;
43pub(crate) use scheduler::{
44    CompactionExecution, CompactionPickFinished, CompactionScheduler, CompactionTransition,
45};
46use serde::{Deserialize, Serialize};
47use snafu::ResultExt;
48use store_api::mito_engine_options::{TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM, TWCS_TRIGGER_FILE_NUM};
49use store_api::storage::RegionId;
50
51use crate::error::{GetSchemaMetadataSnafu, Result, TimeoutSnafu};
52use crate::sst::file::{FileHandle, FileMeta, Level};
53
54/// Finds compaction options and TTL together with a single metadata fetch to reduce RTT.
55async fn find_dynamic_options(
56    region_id: RegionId,
57    region_options: &crate::region::options::RegionOptions,
58    schema_metadata_manager: &SchemaMetadataManagerRef,
59) -> Result<(crate::region::options::CompactionOptions, TimeToLive)> {
60    let table_id = region_id.table_id();
61    if let (true, Some(ttl)) = (region_options.compaction_override, region_options.ttl) {
62        debug!(
63            "Use region options directly for table {}: compaction={:?}, ttl={:?}",
64            table_id, region_options.compaction, region_options.ttl
65        );
66        return Ok((region_options.compaction.clone(), ttl));
67    }
68
69    let db_options = tokio::time::timeout(
70        crate::config::FETCH_OPTION_TIMEOUT,
71        schema_metadata_manager.get_schema_options_by_table_id(table_id),
72    )
73    .await
74    .context(TimeoutSnafu)?
75    .context(GetSchemaMetadataSnafu)?;
76
77    let ttl = if let Some(ttl) = region_options.ttl {
78        debug!(
79            "Use region TTL directly for table {}: ttl={:?}",
80            table_id, region_options.ttl
81        );
82        ttl
83    } else {
84        db_options
85            .as_ref()
86            .and_then(|options| options.ttl)
87            .unwrap_or_default()
88            .into()
89    };
90
91    let compaction = if !region_options.compaction_override {
92        if let Some(schema_opts) = db_options {
93            let mut map: HashMap<String, String> = schema_opts
94                .extra_options
95                .iter()
96                .filter_map(|(k, v)| {
97                    if k.starts_with("compaction.") {
98                        Some((k.clone(), v.clone()))
99                    } else {
100                        None
101                    }
102                })
103                .collect();
104            // Historical metadata may contain both aliases; prefer the canonical key.
105            if map.contains_key(TWCS_ACTIVE_WINDOW_TRIGGER_FILE_NUM) {
106                map.remove(TWCS_TRIGGER_FILE_NUM);
107            }
108            if map.is_empty() {
109                region_options.compaction.clone()
110            } else {
111                crate::region::options::RegionOptions::try_from_options(region_id, &map)
112                    .map(|o| o.compaction)
113                    .unwrap_or_else(|e| {
114                        error!(e; "Failed to create RegionOptions from map");
115                        region_options.compaction.clone()
116                    })
117            }
118        } else {
119            debug!(
120                "DB options is None for table {}, use region compaction: compaction={:?}",
121                table_id, region_options.compaction
122            );
123            region_options.compaction.clone()
124        }
125    } else {
126        debug!(
127            "No schema options for table {}, use region compaction: compaction={:?}",
128            table_id, region_options.compaction
129        );
130        region_options.compaction.clone()
131    };
132
133    debug!(
134        "Resolved dynamic options for table {}: compaction={:?}, ttl={:?}",
135        table_id, compaction, ttl
136    );
137    Ok((compaction, ttl))
138}
139
140#[derive(Debug, Clone)]
141pub struct CompactionOutput {
142    /// Compaction output file level.
143    pub output_level: Level,
144    /// Compaction input files.
145    pub inputs: Vec<FileHandle>,
146    /// Whether to remove deletion markers.
147    pub filter_deleted: bool,
148    /// Compaction output time range. Only windowed compaction specifies output time range.
149    pub output_time_range: Option<TimestampRange>,
150}
151
152/// SerializedCompactionOutput is a serialized version of [CompactionOutput] by replacing [FileHandle] with [FileMeta].
153#[derive(Debug, Clone, Serialize, Deserialize)]
154pub struct SerializedCompactionOutput {
155    output_level: Level,
156    inputs: Vec<FileMeta>,
157    filter_deleted: bool,
158    output_time_range: Option<TimestampRange>,
159}
160
161/// Returns the printable reason carried by a caught panic payload.
162pub(crate) fn format_panic_reason(payload: &(dyn Any + Send + 'static)) -> String {
163    if let Some(message) = payload.downcast_ref::<&str>() {
164        message.to_string()
165    } else if let Some(message) = payload.downcast_ref::<String>() {
166        message.clone()
167    } else {
168        "unknown panic".to_string()
169    }
170}
mito2/compaction.rs

mito2/
compaction.rs
