mito2/worker/
handle_compaction.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use api::v1::region::compact_request;
16use common_telemetry::{debug, error, info};
17use store_api::logstore::LogStore;
18use store_api::region_request::RegionCompactRequest;
19use store_api::storage::RegionId;
20
21use crate::config::IndexBuildMode;
22use crate::error::RegionNotFoundSnafu;
23use crate::metrics::COMPACTION_REQUEST_COUNT;
24use crate::region::MitoRegionRef;
25use crate::request::{
26    BuildIndexRequest, CompactionCancelled, CompactionFailed, CompactionFinished, OnFailure,
27    OptionOutputTx,
28};
29use crate::sst::index::IndexBuildType;
30use crate::worker::RegionWorkerLoop;
31
32impl<S> RegionWorkerLoop<S> {
33    /// Handles compaction request submitted to region worker.
34    pub(crate) async fn handle_compaction_request(
35        &mut self,
36        region_id: RegionId,
37        req: RegionCompactRequest,
38        mut sender: OptionOutputTx,
39    ) {
40        let Some(region) = self.regions.writable_region_or(region_id, &mut sender) else {
41            return;
42        };
43        COMPACTION_REQUEST_COUNT.inc();
44        let parallelism = req.parallelism.unwrap_or(1) as usize;
45        if let Err(e) = self
46            .compaction_scheduler
47            .schedule_compaction(
48                region.region_id,
49                req.options,
50                &region.version_control,
51                &region.access_layer,
52                sender,
53                &region.manifest_ctx,
54                self.schema_metadata_manager.clone(),
55                parallelism,
56            )
57            .await
58        {
59            error!(e; "Failed to schedule compaction task for region: {}", region_id);
60        } else {
61            info!(
62                "Successfully scheduled compaction task for region: {}",
63                region_id
64            );
65        }
66    }
67
68    /// Handles compaction finished, update region version and manifest, deleted compacted files.
69    pub(crate) async fn handle_compaction_finished(
70        &mut self,
71        region_id: RegionId,
72        mut request: CompactionFinished,
73    ) where
74        S: LogStore,
75    {
76        let region = match self.regions.get_region(region_id) {
77            Some(region) => region,
78            None => {
79                request.on_failure(RegionNotFoundSnafu { region_id }.build());
80                return;
81            }
82        };
83
84        region.version_control.apply_edit(
85            Some(request.edit.clone()),
86            &[],
87            region.file_purger.clone(),
88        );
89
90        let index_build_file_metas = std::mem::take(&mut request.edit.files_to_add);
91
92        // compaction finished.
93        request.on_success();
94
95        // In async mode, create indexes after compact if new files are created.
96        if self.config.index.build_mode == IndexBuildMode::Async
97            && !index_build_file_metas.is_empty()
98        {
99            self.handle_rebuild_index(
100                BuildIndexRequest {
101                    region_id,
102                    build_type: IndexBuildType::Compact,
103                    file_metas: index_build_file_metas,
104                },
105                OptionOutputTx::new(None),
106            )
107            .await;
108        }
109
110        // Schedule next compaction if necessary.
111        let mut pending_ddls = self
112            .compaction_scheduler
113            .on_compaction_finished(
114                region_id,
115                &region.manifest_ctx,
116                self.schema_metadata_manager.clone(),
117            )
118            .await;
119        self.handle_ddl_requests(&mut pending_ddls).await;
120
121        if self.compaction_scheduler.is_compacting(region_id) {
122            return;
123        }
124
125        let now = self.time_provider.current_time_millis();
126        if now - region.last_schedule_compaction_millis()
127            >= self.config.min_compaction_interval.as_millis() as i64
128        {
129            debug!(
130                "minimal compaction interval time {:?} has passed, scheduling next compaction",
131                self.config.min_compaction_interval
132            );
133            if self
134                .compaction_scheduler
135                .schedule_next_compaction(
136                    region_id,
137                    &region.manifest_ctx,
138                    self.schema_metadata_manager.clone(),
139                )
140                .await
141            {
142                region.update_schedule_compaction_millis();
143            }
144        }
145    }
146
147    pub(crate) async fn handle_compaction_cancelled(
148        &mut self,
149        region_id: RegionId,
150        request: CompactionCancelled,
151    ) where
152        S: LogStore,
153    {
154        request.on_success();
155
156        // Reuse the scheduler's finish path to wake pending DDLs after a cooperative stop.
157        let mut pending_ddls = match self.regions.get_region(region_id) {
158            Some(_) => {
159                self.compaction_scheduler
160                    .on_compaction_cancelled(region_id)
161                    .await
162            }
163            None => Vec::new(),
164        };
165
166        self.handle_ddl_requests(&mut pending_ddls).await;
167    }
168
169    /// When compaction fails, we simply log the error.
170    pub(crate) async fn handle_compaction_failure(&mut self, req: CompactionFailed) {
171        error!(req.err; "Failed to compact region: {}", req.region_id);
172
173        self.compaction_scheduler
174            .on_compaction_failed(req.region_id, req.err);
175    }
176
177    /// Schedule compaction for the region if necessary.
178    pub(crate) async fn schedule_compaction(&mut self, region: &MitoRegionRef) {
179        if region.is_staging() || region.is_enter_staging() {
180            info!(
181                "Region {} is staging or entering staging, skip compaction",
182                region.region_id
183            );
184            return;
185        }
186        let now = self.time_provider.current_time_millis();
187        if now - region.last_schedule_compaction_millis()
188            >= self.config.min_compaction_interval.as_millis() as i64
189        {
190            debug!(
191                "minimal compaction interval time {:?} has passed, scheduling next compaction",
192                self.config.min_compaction_interval
193            );
194            match self
195                .compaction_scheduler
196                .schedule_compaction(
197                    region.region_id,
198                    compact_request::Options::Regular(Default::default()),
199                    &region.version_control,
200                    &region.access_layer,
201                    OptionOutputTx::none(),
202                    &region.manifest_ctx,
203                    self.schema_metadata_manager.clone(),
204                    1, // Default for automatic compaction
205                )
206                .await
207            {
208                Ok(true) => region.update_schedule_compaction_millis(),
209                Ok(false) => {}
210                Err(e) => {
211                    error!(e; "Failed to schedule compaction for region: {}", region.region_id)
212                }
213            }
214        }
215    }
216}
mito2/worker/handle_compaction.rs

mito2/worker/
handle_compaction.rs
